{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "# IMG_FILEPATH = \"../../01_deep-learning-from-scratch/images/04_extensions/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tanh and derivative"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEWCAYAAACjYXoKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXyddZ3+/9eVpOm+73Qve8vWEsumIgilCENRcURUQERm5iszOuiM4ILzQ52fy3xldMTBqghugIBKGcq0ZZ8Rwba0dKWlC3RvQ5d0TZvl/f3j3IHTkLRJk5P7JOd6Ph7nce77cy/nfdqmV+7PvXwUEZiZmbVEUdoFmJlZ++cwMTOzFnOYmJlZizlMzMysxRwmZmbWYg4TMzNrMYeJWTNJ+ltJT6bwuSdI2pmjfX9a0mO52LcVBoeJdUiS9mS9aiXtz5r/eNr1NYWkzZLeXTcfESsiok8r7PckSdXZbRHx84j4q5bu2wpXSdoFmOVCRPSom5b0OnBjRLT50YRZofCRiRUkSedJeknSTkkbJd0pqSRZ1kVSSLpJ0ipJOyTd+c5d6IfJ9qskXXSYz7pd0hpJuyUtlnRZveX/R9KryfJFkk6V9BAwCJiVHE39Q/YRhaTrJP1vvf3cJul3yfQHJb0iaZektZK+nLXq80Bx1pHahPpdd5LOl/SypApJL0p6V9ayFyV9PXnfJWmGpL7N+guwDsdhYoWqCrgZ6A+8B/gr4MZ660wBJgATgU9Jel/WsvcCc5PtfwT87DCftRw4F+gNfAd4QNIAAEmfBL4EfAzoBVwF7IiIjwBbgckR0SMiflhvn38AJkoamdV2DfDbZHpXMt8HuBL4oqQpWbXXJPvtERHzs3csaRDwGPDt5PvdDcyQ1LveZ30cGJp8xucO8/2tADhMrCBFxF8iYk5E1ETEKjJhcH691f41InZFxBoyv82fkbVseUT8MiJqgPuAUZIaPJ8REQ9GxKaIqI2IXwEbgDOTxTcmnzM/MpZHxPom1L8LmAFcDSDpVGB40kZEPBURS5LPfBn4XQPfrzFTgQUR8buIqI6Ie4H1wKVZ6/w0IlZFxF7gYQ79s7EC5DCxgiRpnKQnJG2RtAu4HRhQb7XNWdP7gB6HWUa95dmf9WlJC5MusZ3AcVmfNQJYdZRf47dkjmggc6TwcEQcTD7zPEnPSSqXVAFczzu/X2OOAd6o1/YGMCxr/nB/NlaAHCZWqH4KvAwcGxG9gDsAtfaHSDoB+A/gJqBfcjXWyqzPWgcc28jmR3qk9wxgjKSTyRyh/DZr2e+AB4EREdEbuDfrM4+0343AqHptI8kcUZk1yGFihaonUBEReySNBz6To8/pAdQC5UCRpL8lc2RS52fArZJOV8YJkoYny7YAYxvbcURUkjl38kOgE/AcZK4MSD53W0RUSjoX+EjWplvJnIAfScOmAxMkXSWpRNK1ZMLkiWZ9cysoDhMrVP8I3ChpD3AXmd/iW11yvuJuMifrNwFjkum65b8Cvk/mvMPu5L3u3Mu3gG8l3WM3N/IRvwUuAh6MiNpknwH8LfBvknYD/ww8lPWZO4DvAvOSfR9yviMitgBXAF8BtpG5UOHyiKg42j8H6/jkwbHMzKylfGRiZmYt5jAxM7MWc5iYmVmLOUzMzKzFCvZBjwMGDIjRo0enXYaZWbsyb968NyNiYP32gg2T0aNHM3fu3COvaGZmb5FU/+kIgLu5zMysFThMzMysxRwmZmbWYg4TMzNrMYeJmZm1WN6EiaR7JG2VtLiR5XXDpK5MxoaYmLXsOkmvJa/r2q5qMzODPAoTMuMtTDnM8kuB45PXTcB/AkjqB3wdOAuYBHzd41GbmbWtvLnPJCKelzT6MKtMBX6ZPF77RUl9JA0F3gfMjojtAJJmkwml+3NbsZm1dxHBwZpaKqtqOVBdw4GqWg5U11JdW0t1TVBTG1TX1r3XUlsL1bW19dqDmmT9CAjq3qE23p4mgsi8HdJe9+T2t9qT6br91NVZG9l1Z01njXVW/yHwh8xmLbzu3NH079G5xX9+2fImTJpgGJlR6eqsT9oaa38HSTeROaph5MjGxgUys/agtjbYse8g2/Ye5M3dB3hz70G27znAngPV7K6sZldldTJdxe7KavZUVrOvqvqtwKisquFAdW3aX6NNKRlr84ozhhV0mLRYREwDpgGUlZV5IBezPFZbG6zbsY81b+5l3Y79rN++j/U79rNuxz427qxkx76D1NQ2/GPcuaSInl1K6NmlEz27lNCjcwn9+3eje+cSOpcU0aVTMZ1LijKvuulOxXQpKaK0pIjS4iKKi0RJsSiSKCl6e764SJQU1b0XHTIvgSQEFCmZBxCIzHxRslxJG+Kt9d7eRm/9x3/oNm+PLJ09xnRW8yHrtKX2FCYbgBFZ88OTtg1kurqy259ts6rMrMUOVNeweMMuFqzbyfLNu1i+eTcrtuxhf1XNW+uUFhcxrG9XhvftyrihvRjQozP9e5QyoEfn5FVKv+6l9OzSidKSfDodXBjaU5hMB26W9ACZk+0VEbFJ0kzgX7NOuk8GbkurSDM7sgPVNcxZs4P/ea2cuW/sYNH6Cg7WZLqcBvQo5YTBPbl60ghOHNyTYwf1YETfbgzq2ZmionR+67Yjy5swkXQ/mSOMAZLWk7lCqxNARNwNzAA+AKwE9gGfSpZtl/QNYE6yqzvqTsabWf6o2F/Ffy/exJPLtvKnlW+y72ANpcVFnDq8N9efN5qJI/sycWQfBvXqknapdhQKdgz4srKy8FODzXKruqaWp1/dyh/mb+CpV7dysLqWYX26csFJA7ngxEGcc2x/upXmze+01gSS5kVEWf12/y2aWaur2FfF/XPW8ssXXmdjRSUDepTy8bNG8sEJwzh1WO/UThJb7jhMzKzV7Nh7kLufX8UvX3iD/VU1nDO2P/9yxXguPGkQJcU+Kd6ROUzMrMUqq2qY9vxqfvr8avYcrOaK04/hb957LOOO6ZV2adZGHCZm1iLPLN/K1x9dwtrt+5g8bjBfmHwiJw7pmXZZ1sYcJmZ2VCr2VfHVRxfz2CsbGTuwO7+98SzOPW5A2mVZShwmZtZsL67exi0PLmDr7gPccvEJ/M35Y+lcUpx2WZYih4mZNVlEcPdzq/nuzFcZ3b87j/zduZw+ok/aZVkecJiYWZNUVtXwpUcW8uiCjVx22lC+++HT6N7Z/4VYhv8lmNkRVeyv4oZ75zDvjR18cfIJfPaC43yviB3CYWJmh/XmngNc+/O/8NrW3dx1zUQuO21o2iVZHnKYmFmjtu6u5OppL7Jx535+em0Z7ztxUNolWZ5ymJhZgyr2V3Htz//Cpp2V/PKGs5g0pl/aJVke8/MNzOwd9h2s5oZ757CqfA/Trj3TQWJH5DAxs0PU1gafe2AB89fu4AdXT+A9xw9MuyRrBxwmZnaIO59cweylW/ja5eP4wKk+2W5N4zAxs7c8vnAT//H0Sj5aNoLrzx2ddjnWjuRNmEiaImm5pJWSbm1g+Z2SFiSvFZJ2Zi2ryVo2vW0rN+sYVm7dwxcfeoUzR/XljivH+z4Sa5a8uJpLUjFwF3AxsB6YI2l6RCytWyci/jFr/b8HJmTtYn9EnNFW9Zp1NAeqa/iH++fTpVMRP/74RD9ny5otX45MJgErI2J1RBwEHgCmHmb9jwH3t0llZgXgO08sZ+mmXXzvqtMZ7DHY7SjkS5gMA9Zlza9P2t5B0ihgDPB0VnMXSXMlvSjpysY+RNJNyXpzy8vLW6Nus3bvuRXl3POnNVx3ziguGjc47XKsncqXMGmOq4GHI6Imq21UMsD9NcC/Szq2oQ0jYlpElEVE2cCBvtzRbM+Bar78+0UcO7A7t33g5LTLsXYsX8JkAzAia3540taQq6nXxRURG5L31cCzHHo+xcwa8W8zl7OxYj/fveo0unTyeRI7evkSJnOA4yWNkVRKJjDecVWWpJOAvsCfs9r6SuqcTA8AzgOW1t/WzA718tod3Pfn1/nk2aM4c5TvcLeWyYuruSKiWtLNwEygGLgnIpZIugOYGxF1wXI18EBERNbmJwM/kVRLJhy/nX0VmJm9U3VNLV/+/SKG9OrCP085Ke1yrAPIizABiIgZwIx6bbfXm/+XBrZ7ATg1p8WZdTD3/2Utr27ezd2fmEgPD3BlrSBfurnMrI1U7Kvi+7NXcM7Y/lwyfkja5VgH4TAxKzA/eOo1KvZX8bXLx/kud2s1DhOzArKqfA+//PPrfPRdIxl3TK+0y7EOxGFiVkDunL2C0pIivjD5hLRLsQ7GYWJWIJZu3MV/LdzEDeeNYUCPzmmXYx2Mw8SsQHx/9gp6dinhM+8Zm3Yp1gE5TMwKwIJ1O3ly2Rb+5r1j6d2tU9rlWAfkMDErAHfOXkG/7qVcf96YtEuxDsphYtbBLd5QwXMryvn0u8f4BkXLGYeJWQd393Or6Nm5hE+eMyrtUqwDc5iYdWCvv7mXGYs28fGzR9Gri8+VWO44TMw6sGn/s5qSoiJuOG902qVYB+cwMeugtu6u5OF56/nwmcMZ5KF4LcccJmYd1G9fWsvB6lo+8x5fwWW55zAx64AOVtfym5fWcv4JAxk7sEfa5VgByJswkTRF0nJJKyXd2sDy6yWVS1qQvG7MWnadpNeS13VtW7lZ/nli8SbKdx/gep8rsTaSFxedSyoG7gIuBtYDcyRNb2DExAcj4uZ62/YDvg6UAQHMS7bd0Qalm+Wl+154nTEDunP+8QPTLsUKRL4cmUwCVkbE6og4CDwATG3itpcAsyNiexIgs4EpOarTLO8tWl/By2t38smzR1FU5PFKrG3kS5gMA9Zlza9P2ur7sKSFkh6WNKKZ25oVhHtfeJ1upcVcVTY87VKsgORLmDTFY8DoiDiNzNHHfc3dgaSbJM2VNLe8vLzVCzRL2859B3ls4UY+NHGYb1K0NpUvYbIBGJE1Pzxpe0tEbIuIA8nsz4Azm7pt1j6mRURZRJQNHOi+ZOt4/jh/Awera7lmkh+dYm0rX8JkDnC8pDGSSoGrgenZK0gamjV7BbAsmZ4JTJbUV1JfYHLSZlZQIoIH5qzjtOG9PSSvtbm8uJorIqol3UwmBIqBeyJiiaQ7gLkRMR34B0lXANXAduD6ZNvtkr5BJpAA7oiI7W3+JcxStnB9Ba9u3s23PnhK2qVYAcqLMAGIiBnAjHptt2dN3wbc1si29wD35LRAszz34Nx1dO1UzBWnH5N2KVaA8qWby8xaYN/BaqYv2Mhlpw2lp0+8WwocJmYdwOMLN7HnQDVXv2vEkVc2ywGHiVkH8NC89Ywd2J0zR/VNuxQrUA4Ts3Zu3fZ9/GXNdj48cTiS73i3dDhMzNq56a9sBPCJd0uVw8SsHYsIfv/yeiaN7seIft3SLscKmMPErB1bvGEXq8r3cuUEP47O0uUwMWvH/jB/A6XFRVx26tAjr2yWQw4Ts3aquqaW6a9s5MKTBtG7m+8tsXQ5TMzaqT+t2sabew64i8vygsPErJ364/wN9OpSwgUn+QnYlj6HiVk7VFlVw6wlm7n0lKF0LilOuxwzh4lZe/TcinL2HqzhstN84t3yg8PErB16fOEm+nbrxDnH9k+7FDPAYWLW7lRW1fDUsi1MOWUInYr9I2z5wf8SzdqZZ5cnXVyn+vEplj/yJkwkTZG0XNJKSbc2sPwWSUslLZT0lKRRWctqJC1IXtPrb2vWkTy+aBP9updy9th+aZdi9pa8GGlRUjFwF3AxsB6YI2l6RCzNWm0+UBYR+yT9HfBd4KPJsv0RcUabFm2Wgv0HM11cV04YRom7uCyP5Mu/xknAyohYHREHgQeAqdkrRMQzEbEvmX0RGN7GNZql7tnlW9l3sIbL/fgUyzP5EibDgHVZ8+uTtsZ8Gngia76LpLmSXpR0ZWMbSbopWW9ueXl5yyo2S8HjizbRv3spk8a4i8vyS150czWHpE8AZcD5Wc2jImKDpLHA05IWRcSq+ttGxDRgGkBZWVm0ScFmrSTTxbWVD010F5fln3z5F7kByB68enjSdghJFwFfAa6IiAN17RGxIXlfDTwLTMhlsWZpeHb5VvZX1fgJwZaX8iVM5gDHSxojqRS4GjjkqixJE4CfkAmSrVntfSV1TqYHAOcB2SfuzTqEmUs2089dXJan8qKbKyKqJd0MzASKgXsiYomkO4C5ETEd+B7QA3goGed6bURcAZwM/ERSLZlw/Ha9q8DM2r2qmlqeenUrU8YPcReX5aW8CBOAiJgBzKjXdnvW9EWNbPcCcGpuqzNL10urt7O7sprJ44ekXYpZg/wrjlk7MGvpZrp2KuY9xw9IuxSzBjlMzPJcRDBryRbee8IAunTy4+YtPzU7TCR1T+5YN7M2sHB9BZt3VTJ5nLu4LH8dMUwkFUm6RtLjkrYCrwKbkudkfU/Scbkv06xwzVq6meIi8f6TB6VdilmjmnJk8gxwLHAbMCQiRkTEIODdZB5r8p3kRkIzy4FZS7Zw1ph+9OlWmnYpZo1qytVcF0VEVf3GiNgOPAI8IqlTq1dmZqwu38NrW/fw8bNGpl2K2WEd8cikLkgk/UDJDR6NrWNmrWv20i0AXOxLgi3PNecE/G5guqTuAJIukfSn3JRlZgCzlm7hlGG9GNana9qlmB1Wk29ajIivSroGeFbSQWAP8I5BrMysdWzdXcnLa3fwjxedkHYpZkfU5DCR9H7gM8BeYChwQ0Qsz1VhZoXuqWVbiYDJ4wenXYrZETWnm+srwNci4n3AVcCDki7MSVVmxqwlmxnZrxsnDu6ZdilmR9TkMImICyPif5PpRcClwDdzVZhZIdtdWcWfVm5j8rjBNHLdi1leacpNi41dwbUJeP/h1jGzo/PcinIO1tT6wY7WbjTppkVJfy/pkAvdk3FHzpF0H3BdTqozK1CzlmyhX/dSzhzVN+1SzJqkKSfgpwA3APcnw+LuALqSCaJZwL9HxPzclWhWWA5W1/LMq1u59NQhFBf5oN/ahyOGSURUAj8Gfpzc6T4A2B8RO3NdnFkhenH1NnYfqOYSd3FZO9LkE/DJlVt3A18EPijpzLrhcluDpCmSlktaKekd969I6izpwWT5S5JGZy27LWlfLumS1qrJLA2zlm6mW2kx5x3nsUus/WjOpcH3AI+RebjjWOB2YElrFJE80v4uMleIjQM+JmlcvdU+DeyIiOOAO4HvJNuOIzNm/HgyXXI/9iPyrb2qrQ1mL93C+ScM9Ngl1q40Z9jeNyLij8n0Q61cxyRgZUSsBpD0ADAVyB7LfSrwL8n0w8CPkqvIpgIPRMQBYI2klcn+/tzKNZrl3MINFWzZdcA3Klq705RLg38p6fPAi5JuyVEdw4B1WfPrk7YG14mIaqAC6N/EbQGQdJOkuZLmlpeXt1LpZq1n1pLM2CUXnugwsfalKd1c9wICBgOflPSGpOmSviHpIzmtrpVFxLSIKIuIsoEDB6Zdjtk7zFyymbPH9qN3N4/qYO1LU67mehp4um5eUglwMnA6me6k1ujy2gCMyJofnrQ1tM76pIbewLYmbmuW91Zu3cOq8r1ce87otEsxa7ZmXc0l6edkTnyXAcuAr7ZSHXOA4yWNSW6GvBqYXm+d6bx9c+RVwNMREUn71cnVXmOA44G/tFJdZm3mrbFLxrmLy9qf5pyAvwf4PNAJOA24kswVVC0eAz4iqiXdDMwEioF7ImKJpDuAuRExHfg58KvkBPt2MoFDst7vyJysrwY+GxE1La3JrK3NWrqZU4b14hiPXWLtUL5czUVEzABm1Gu7PWu6EmjwHE1EfAv4VmvXZNZWtu6qZP7anXzhYo9dYu1Tc+4zeV7SP/qhjmatb/ayTBfXJaf4rndrn5pzZDIOOBX4kqR5wAJgQUS0+lGKWaGZuWQLo/t34/hBPdIuxeyoNGfY3g8DSOrK28FyNjno8jIrJLsqq/jzqjf51HljPHaJtVvNGba3P/DXQCWZx6g8GBH35qgus4Lx7PJyqmqCyb6Ky9qx5pwz+QMwEPhX4HtAhaRlOanKrIDMWrKZAT06M2Gkxy6x9qs5YdIzIu4AtkTE+cDHcBeXWYscqK7h2eXlXDxukMcusXatOWFSmbwfkNQ1Ih4BJuegJrOC8cKqbew5UM3kcb6Ky9q35lzN9W+S+gEPAvdIegHok5uyzArDrCVb6F5azDnH9k+7FLMWacpTg/8vQEQ8EhHbI+L7ZG4uHAF8KMf1mXVYdWOXvO/EQR67xNq9phyZXFC/ISJ+lYNazArK/HU7eXOPxy6xjqE550zMrBXNWrqZTsXigpMGpV2KWYs15cjkdElrgEXA4qz3ZckgVWbWTBHBrCVbOHtsf3p18dgl1v415chkIXAe8CMy44dMBn4BbJO0OIe1mXVYK7fuYc2be5k83ldxWcfQpKu5ImIjsBGYVdeWPPCxxY+fNytEM5dsBvBd79ZhNCVMftRQYzIw1WutW45ZYZixaDNnjurL4F5d0i7FrFUcsZsrIn7eFoWYFYrX39zL0k27uNSPm7cOJPWruST1kzRb0mvJ+zseUCTpDEl/lrRE0kJJH81adq+kNZIWJK8z2vYbmDXPjMWbALj01KEpV2LWelIPE+BW4KmIOB54Kpmvbx9wbUSMB6YA/y4p++77f4qIM5LXgtyXbHb0nli0mTNG9GGYh+e1DiQfwmQqcF8yfR+ZseUPERErIuK1ZHojsJXME4zN2pW12/axaEMFl/moxDqYfAiTwRGxKZneDBz28hZJk4BSYFVW87eS7q87JXU+zLY3SZoraW55eXmLCzdrrieSLq4pPl9iHUybhImkJyUtbuA1NXu95AqxOMx+hgK/Aj4VEbVJ823AScC7gH7AlxrbPiKmRURZRJQNHOgDG2t7MxZt4rThvRnRr1vapZi1quY8NfioRcRFjS2TtEXS0IjYlITF1kbW6wU8DnwlIl7M2nfdUc0BSb8AvtiKpZu1mvU79vHK+gpuvfSktEsxa3X50M01Hbgumb4OeLT+CpJKyYz0+MuIeLjesqHJu8icb/Fd+ZaXnliUuVHxA6f4fIl1PPkQJt8GLpb0GnBRMo+kMkk/S9b5a+C9wPUNXAL8G0mLyDwzbADwzbYt36xpZizexCnDejGyv7u4rONpk26uw4mIbcD7G2ifC9yYTP8a+HUj21+Y0wLNWsGGnfuZv3Yn/3TJiWmXYpYT+XBkYtbhPfbKRgD+6rRjUq7ELDccJmZt4I/zNzBhZB93cVmH5TAxy7Hlm3fz6ubdXHnGsLRLMcsZh4lZjk1/ZQPFReIDvuvdOjCHiVkORQSPLtjIeccNYGDPRh/OYNbuOUzMcujltTtYv2M/U0/3iXfr2BwmZjn06IKNdC4pYvJ4j6hoHZvDxCxHqmpqeXzhJi4aN5ieXTqlXY5ZTjlMzHLk+RXlbNt70F1cVhAcJmY58tDc9fTvXsoFJw1KuxSznHOYmOXAtj0HeOrVLXxwwjA6FfvHzDo+/ys3y4E/LthIVU3wkbIRaZdi1iYcJmatLCJ4aO46ThvemxOH9Ey7HLM24TAxa2VLNu7i1c27+ciZw9MuxazNOEzMWtnv5q6jtKSIK073s7iscKQeJpL6SZot6bXkvW8j69VkDYw1Pat9jKSXJK2U9GAyKqNZKiqranh0wUYuGT+E3t18b4kVjtTDBLgVeCoijgeeSuYbsj8izkheV2S1fwe4MyKOA3YAn85tuWaNe+yVjVTsr+LjZ41MuxSzNpUPYTIVuC+Zvo/MOO5Nkoz7fiFQNy58s7Y3a22/fvENjh/Ug7PG9Eu7FLM2lQ9hMjgiNiXTm4HGHmLURdJcSS9KqguM/sDOiKhO5tcD7qi2VCxcv5NX1lfwibNHkfk9x6xwtMkY8JKeBIY0sOgr2TMREZKikd2MiogNksYCT0taBFQ0s46bgJsARo50N4S1rl+/+AbdSov54ET/PmOFp03CJCIuamyZpC2ShkbEJklDga2N7GND8r5a0rPABOARoI+kkuToZDiw4TB1TAOmAZSVlTUWWmbNVrGvikcXbOTDZw6nlx/qaAUoH7q5pgPXJdPXAY/WX0FSX0mdk+kBwHnA0ogI4BngqsNtb5ZrD81bx4HqWj5x1qi0SzFLRT6EybeBiyW9BlyUzCOpTNLPknVOBuZKeoVMeHw7IpYmy74E3CJpJZlzKD9v0+qt4FXX1HLfn1+nbFRfxh3TK+1yzFLRJt1chxMR24D3N9A+F7gxmX4BOLWR7VcDk3JZo9nhzFyyhXXb9/PVy8alXYpZavLhyMSs3YoIpj2/ijEDunPRyR5N0QqXw8SsBea8voNX1lfw6XePobjIlwNb4XKYmLXAtOdX0697KR+e6Ic6WmFzmJgdpRVbdvPksi184uxRdC0tTrscs1Q5TMyO0g+feo3upcV86tzRaZdiljqHidlReG3Lbh5ftIlrzx1N3+5+ULWZw8TsKPzw6ZV07VTMZ94zNu1SzPKCw8SsmVZu3c1/LdzIteeMpp+PSswAh4lZs31/9orkqGRM2qWY5Q2HiVkzzHtjBzMWbeYz7xlL/x6d0y7HLG84TMyaKCL41xnLGNizMze91+dKzLI5TMyaaOaSzcx7Ywe3XHwC3Tun/lg7s7ziMDFrggPVNXznv5dz/KAefORM3+1uVp/DxKwJpj23mjVv7uWrl4+jpNg/Nmb1+afC7Ahef3Mv//HMSi47bSjnnzAw7XLM8pLDxOwwIoLbpy+htLiI2y/3eCVmjUk9TCT1kzRb0mvJe98G1rlA0oKsV6WkK5Nl90pak7XsjLb/FtZRPbZwE8+vKOeWi09gcK8uaZdjlrdSDxPgVuCpiDgeeCqZP0REPBMRZ0TEGcCFwD5gVtYq/1S3PCIWtEnV1uFtrqjka39czOkj+nDtOR7b3exw8iFMpgL3JdP3AVceYf2rgCciYl9Oq7KCFhH88yMLOVBdw51/fbpPupsdQT78hAyOiE3J9GbgSGOfXg3cX6/tW5IWSrpTUqO3JUu6SdJcSXPLy8tbULJ1dL9+8Q2eX1HOlz9wMmMH9ki7HLO81yZhIulJSYsbeE3NXi8iAojD7GcocCowM6v5NuAk4F1AP+BLjW0fEdMioiwiygYO9FU51rDFGyr45uPLeM/xA/jk2e7eMmLK77kAAAkiSURBVGuKNrmNNyIuamyZpC2ShkbEpiQsth5mV38N/CEiqrL2XXdUc0DSL4AvtkrRVpB27jvI3/1mHn27lXLnR89A8rjuZk2RD91c04HrkunrgEcPs+7HqNfFlQQQyvzUXwkszkGNVgBqa4PPP7iAzRWV/PgTExngBzmaNVk+hMm3gYslvQZclMwjqUzSz+pWkjQaGAE8V2/730haBCwCBgDfbIOarQP6/59YxrPLy7n98nFMHPmOK9TN7DBSf1pdRGwD3t9A+1zgxqz514FhDax3YS7rs8Jwz/+u4af/s4ZrzxnFJ3yexKzZ8uHIxCxVjy/cxDceX8ol4wfz9b8a7/MkZkfBYWIF7YlFm/jcA/M5c2RffnD1BIqLHCRmR8NhYgXr8YWbuPn++Zw+og+/+NS76NKpOO2SzNqt1M+ZmKXhty+t5WuPLmbCiD7ce8MkeniwK7MW8U+QFZTa2uA7M1/lJ8+t5n0nDuSuayZ61ESzVuCfIisYFfuq+OLDrzB76RY+ftZI/r8rxvuZW2atxGFiBeGVdTv57G9fZnNFJbdfPo5PnTfaV22ZtSKHiXVoB6pr+NHTK/nPZ1cxuFcXHvrbc5jgGxLNWp3DxDqsOa9v57bfL2Ll1j18aOIwbr98HH26laZdllmH5DCxDmfNm3v57n+/yhOLNzOsT1fuu2GSx243yzGHiXUYq8r38NPnV/PwvPWUlhRxy8UncON7xtCt1P/MzXLNP2XWrtXWBi+t2c4v/rSG2cu2UFpcxDVnjeTmC49jUE+P2W7WVhwm1i6t276PP87fwEPz1rN2+z56d+3E319wHNeeO9qPjjdLgcPE2oXa2mDppl08uWwLM5dsYdmmXQCcM7Y/t1x8ApeMH0LXUj8OxSwtDhPLSweqa1ixeQ9/eX07L67exl/WbKdifxUSlI3qy1cvO5lLxg9hRL9uaZdqZjhMLGU1tcGGHftZs20vq8v3sHTjLpZs3MWKLbuprg0ARvXvxpTxQzj72H68+7iBDOzpbiyzfJN6mEj6CPAvwMnApGRQrIbWmwL8ACgGfhYRdSMyjgEeAPoD84BPRsTBNijdjuBAdQ0V+6rYuvsAW3ZVsmVX5n3r7ko2V1TyxvZ9rNu+j6qaeGub/t1LGT+sN+efOJDxx/Ri4si+HNOna4rfwsyaIvUwITNm+4eAnzS2gqRi4C7gYmA9MEfS9IhYCnwHuDMiHpB0N/Bp4D9zX3b7UVsbVNcGNbVBdW0t1TVNmz9YXUtlVQ2VVTXsr6qhsqo2ea85pH3vgRoq9ldRsb+KXcl7xf4qDlTXvqMWCfp378zgXp05YVBPJo8bwpgB3RjdvztjBnRnYM/OfsyJWTuUephExDLgSP+BTAJWRsTqZN0HgKmSlgEXAtck691H5ignZ2Hy5T8s4qXV2wiA5BfqACIiea9rCyLenqfeOnXL394+2epw+0zmibeXZ29ff58EVNfWUptVQ2vq2qmYLp2K6N65hN5dO9G7ayeOHdgjM90tM9+rSwkDe3ZhcK/ODOndhQE9OtPJD1c063BSD5MmGgasy5pfD5xFpmtrZ0RUZ7W/Y5z4OpJuAm4CGDly5NEV0qcrJw3pBQLxdghmpt/ZlllP1GXl2+tltSUrvr193bp6xz5paPu6Numtz6zbvlOxKC4SnYqLKC4SJUWZ+ZLiorenixpep7SkKAmM4kPeO3cqonNJkY8gzOwtbRImkp4EhjSw6CsR8Whb1AAQEdOAaQBlZWVH9fv6Zy84rlVrMjPrCNokTCLiohbuYgMwImt+eNK2DegjqSQ5OqlrNzOzNtReOq/nAMdLGiOpFLgamB6ZkwbPAFcl610HtNmRjpmZZaQeJpI+KGk9cA7wuKSZSfsxkmYAJEcdNwMzgWXA7yJiSbKLLwG3SFpJ5hzKz9v6O5iZFTpF5OhSnzxXVlYWc+c2eEuLmZk1QtK8iCir3576kYmZmbV/DhMzM2sxh4mZmbWYw8TMzFqsYE/ASyoH3ki7jqMwAHgz7SLaUKF9X/B3LhTt9TuPioiB9RsLNkzaK0lzG7qSoqMqtO8L/s6FoqN9Z3dzmZlZizlMzMysxRwm7c+0tAtoY4X2fcHfuVB0qO/scyZmZtZiPjIxM7MWc5iYmVmLOUzaKUlfkBSSBqRdS65J+p6kVyUtlPQHSX3SrilXJE2RtFzSSkm3pl1PrkkaIekZSUslLZH0ubRraiuSiiXNl/RfadfSGhwm7ZCkEcBkYG3atbSR2cApEXEasAK4LeV6ckJSMXAXcCkwDviYpHHpVpVz1cAXImIccDbw2QL4znU+R2ZIjQ7BYdI+3Qn8M1AQV09ExKxkTBuAF8mMqNkRTQJWRsTqiDgIPABMTbmmnIqITRHxcjK9m8x/rsPSrSr3JA0HLgN+lnYtrcVh0s5ImgpsiIhX0q4lJTcAT6RdRI4MA9Zlza+nAP5jrSNpNDABeCndStrEv5P5hbA27UJaS5uMAW/NI+lJYEgDi74CfJlMF1eHcrjvHBGPJut8hUy3yG/asjbLPUk9gEeAz0fErrTrySVJlwNbI2KepPelXU9rcZjkoYi4qKF2SacCY4BXJEGmu+dlSZMiYnMbltjqGvvOdSRdD1wOvD867s1RG4ARWfPDk7YOTVInMkHym4j4fdr1tIHzgCskfQDoAvSS9OuI+ETKdbWIb1psxyS9DpRFRHt88miTSZoCfB84PyLK064nVySVkLnA4P1kQmQOcE1ELEm1sBxS5rei+4DtEfH5tOtpa8mRyRcj4vK0a2kpnzOx9uBHQE9gtqQFku5Ou6BcSC4yuBmYSeZE9O86cpAkzgM+CVyY/N0uSH5jt3bGRyZmZtZiPjIxM7MWc5iYmVmLOUzMzKzFHCZmZtZiDhMzM2sxh4mZmbWYw8TMzFrMYWKWJ5JxPS5Opr8p6T/SrsmsqfxsLrP88XXgDkmDyDw994qU6zFrMt8Bb5ZHJD0H9ADel4zvYdYuuJvLLE8kT4UeChx0kFh74zAxywOShpIZp2UqsCd5UrJZu+EwMUuZpG7A78mMhb4M+AaZ8ydm7YbPmZiZWYv5yMTMzFrMYWJmZi3mMDEzsxZzmJiZWYs5TMzMrMUcJmZm1mIOEzMza7H/Bxy/ngGAV1qnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = np.arange(-5, 5, 0.01)\n",
    "plt.plot(a, np.tanh(a))\n",
    "plt.title(\"Tanh activation\")\n",
    "plt.xlabel(\"$x$\")\n",
    "plt.ylabel(\"$Tanh(x)$\");\n",
    "# plt.savefig(IMG_FILEPATH + \"05_tanh_activation.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9999999999999787"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(a[501] - a[500]) / 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAfN0lEQVR4nO3deXRV5b3/8feXhCRkIkASkCQQkIRBBoEIjmhVKlov1IpWvXqrVemvlo7cttapDtVVtbWjHaxtHepEFZUqjtVOVpAIyDyEMCSBzCHzfJ7fH4neSMEc4CT7DJ/XWlk5Z5+d5HMgfNbDs5+9tznnEBGR0DfA6wAiIhIYKnQRkTChQhcRCRMqdBGRMKFCFxEJE9Fe/eDU1FSXnZ3t1Y8XEQlJ77//fqVzLu1Qr3lW6NnZ2eTn53v140VEQpKZ7Tnca5pyEREJEyp0EZEwoUIXEQkTKnQRkTDRa6Gb2R/MrNzMNh7mdTOzn5tZgZmtN7MZgY8pIiK98WeE/ggw7xNePx/I6f5YBPz62GOJiMiR6rXQnXP/AKo/YZcFwGOuy0ogxcyOC1RAERHxTyDWoWcART2eF3dv23/wjma2iK5RPKNGjQrAjxYRCR5tHT7qWtqpbW6noaWDxtYOGts6aWztoKG1g6a2DhpaOzl3YjpTM1MC/vP79cQi59xDwEMAeXl5uhC7iAQl5xz1rR1UNbRR2dBKVUMrFQ1tVDW0cqCpnbrmrtL+sLxrm9upa+6gub3Tr++fnhQbtIVeAmT1eJ7ZvU1EJOi0dfgoq2th34Fm9te2sK+2mf0HWthf20x5fSuV9a1UNrbR1uE75NcnxUUzeNBAkuMGMnjQQMakJnzsefKgrs9JcdHEx0STGBtNfGwUibHRJMRGEz8wigEDrE/eWyAKfTmw2MyeBmYDtc65/5huERHpLw2tHeypamR3ZRO7qxrZXdnI7qpG9lQ1UdHQysE3akuOi2ZkyiCGJ8eRk55EamIMqYmxDOvxOS0xliEJMQyMCt7V3r0Wupk9BZwFpJpZMfB9YCCAc+43wArgAqAAaAKu6auwIiI9Nbd1sqO8nq3769laWs+2sjq2lzVQUd/6sf3SkmIZMyyBM3PTGJkyiJEpcRw3+P8+J8R6dlmrgOr1XTjnLu/ldQd8JWCJREQOobmtk437alm39wDrig+weV8du6saPxptx0YPIHd4EnNy0hiblkD2sASyU+PJHpYQNoXdm8h4lyISckprW3i3sJLVu2tYt/cA28rq6fR1tXdGyiCmZAxm/rSRTBiRxPgRSYwelkBUH81NhwoVuogEhfK6Ft4trGJlYRXv7qxid1UT0HUQ8sSsFL484XhOzEphWlYKaUmxHqcNTip0EfGEz+fYUFLLX7eW8/bWcjaU1AJdBT57zDCuPHk0pxw/jIkjkvtsVUi4UaGLSL9p7/Tx751VrFi/n79uLaeyoZUBBjNGDeHb543nzNw0Jh6XHPFTJ0dLhS4ifarT51i9u5q/fLCPVzaWUt3YRlJsNGdNSOecCemcmZvGkIQYr2OGBRW6iPSJouomnlldxLPvF1Na18KggVGcO2k4/zX1OM4cn0ZsdJTXEcOOCl1EAqatw8frm0t5ZnUR/9xRyQCDM3PTuOXCiZw9IZ34GFVOX9Kfrogcs5rGNp58by+P/ns35fWtZKQM4pvn5nJJXiYjUwZ5HS9iqNBF5Kjtqmzk9/8q5Nn3i2lp93FGTir3LpzKnJw0Hdj0gApdRI5YYUUDv3irgBfXlRA9YACfnT6Sa08fy/gRSV5Hi2gqdBHx267KRn7x1x28sK6EmOgBXHv6GK6fM5b0pDivowkqdBHxQ1VDKz99cwdPvreXgVHGtaePYdGc43XGZpBRoYvIYbV2dPLIO7v55VsFNLV3csWsUXztnBwVeZBSoYvIIb25uYw7XtpEUXUznxqfxs2fmci4dM2RBzMVuoh8TGltC7cv38Srm0rJSU/ksS/OYk5umtexxA8qdBEBui6W9adVe7jv1W20d/r49nnjuf6MscREB+8deuTjVOgiQnFNE0uWfsCqXdWckZPKXQsmk52a4HUsOUIqdJEI5pzj2feLueMvmwG4b+FULpmZiZlOCgpFKnSRCFXT2MaNy9bz2qYyZo0Zyo8vmUbW0HivY8kxUKGLRKC1e2tY/ORaKupbuemCCVx7+lidqh8GVOgiEcQ5x2Pv7uEHL29meHIcz375FKZmpngdSwJEhS4SIRpbO/juc+t5af1+zpmQzgOXnsjg+IFex5IAUqGLRICSA81c92g+20rr+O68CXxpzljdpzMMqdBFwtyavTUseux9Wts7+eM1szhTJwmFLRW6SBh7cV0J3352PSOS43jq+tnkDNep++FMhS4Shpxz/PrvO7nv1W3MGjOU31w5k6G6EXPYU6GLhBmfz3HPii08/K9dzJ82kh9dMk2n70cIFbpIGGnv9PHd59azbE0JV5+azW0XTtLBzwiiQhcJEy3tndzwxBre2lrOt+bm8tWzx+kU/gijQhcJAy3tnVz/WD7/Kqjkrs9O5qqTR3sdSTygQhcJcT3L/L6Lp3JJXpbXkcQjfh0pMbN5ZrbNzArM7MZDvD7KzN42s7Vmtt7MLgh8VBE5WHNbJ9c92lXm9y+cpjKPcL0WuplFAQ8C5wOTgMvNbNJBu90CLHXOTQcuA34V6KAi8nEfjszf2dlV5gtnZnodSTzmzwh9FlDgnCt0zrUBTwMLDtrHAcndjwcD+wIXUUQO1t7pY/GTa3hnZyU/UplLN38KPQMo6vG8uHtbT7cDV5pZMbAC+OqhvpGZLTKzfDPLr6ioOIq4IuLzOb777Hre3FLOnQsmc7HKXLoF6myDy4FHnHOZwAXA42b2H9/bOfeQcy7POZeXlqbrSYgcKeccd760mWVrS1gyN1erWeRj/Cn0EqDnkZbM7m09XQssBXDOvQvEAamBCCgi/+fnfy3gkX/v5trTx7D47HFex5Eg40+hrwZyzGyMmcXQddBz+UH77AXOATCziXQVuuZURALo6ff28pM3t3PxjExuvmCiThqS/9BroTvnOoDFwGvAFrpWs2wyszvNbH73bkuA683sA+Ap4GrnnOur0CKR5p87Krj5hY3MyU3j3oun6HR+OSS/Tixyzq2g62Bnz2239Xi8GTgtsNFEBGBbaT03/GkNOemJPHjFdKKjdKEtOTT9ZogEsfK6Fr74yGoGxUTxh6tPIilOt4yTw9Op/yJBqrmtk+sey6e6sY0//79TGJkyyOtIEuRU6CJByDnH95atZ0NJLQ9dlcfkjMFeR5IQoCkXkSD0+3/t4oV1+1gyN5e5k4Z7HUdChApdJMj8a0cl96zYwvmTR/CVT2mtufhPhS4SRPZWNbH4qTWMS0/kR5dM01pzOSIqdJEg0dTWwaLH8/H5HA9dlUdCrA5xyZHRb4xIEHDOcfPzG9lWVs8frz6J7NQEryNJCNIIXSQI/Dm/mOfXlvD1c3I4a3y613EkRKnQRTy2tbSOW1/cyGnjhvHVs3O8jiMhTIUu4qHG1g6+8sQakgcN5Kefn06UrtEix0CFLuIR5xy3vLCRXZWN/OyyE0lLivU6koQ4FbqIR5bmF/H82hK+cW4upx6v2wfIsVOhi3igoLyB7y/fxOnjUnXykASMCl2kn7V1+PjGM2uJj4nmgUunad5cAkbr0EX62U/e3M7Gkjp+e9VM0pPjvI4jYUQjdJF+tLKwit/8fSeXnZTFeSeM8DqOhBkVukg/qW1uZ8nSDxg9NJ5bL5zkdRwJQ5pyEeknt724kdK6Fp778qm6Tov0CY3QRfrBi+tKeHHdPr5+Tg4nZqV4HUfClApdpI+V1bVw6wsbmTEqhRvOOt7rOBLGVOgifajrVnIbaOv08eNLTyQ6Sv/kpO/ot0ukDy1bU8JbW8v59nkTGKNL4kofU6GL9JGyuhbu+MsmTsoewjWnZnsdRyKACl2kD/Scarlv4TQG6GxQ6QcqdJE+oKkW8YIKXSTANNUiXlGhiwSQplrESyp0kQB6cd0+TbWIZ1ToIgFS29TOD17ezLSsFK7WVIt4QIUuEiD3vbaV6sY27v7sZF3jXDzhV6Gb2Twz22ZmBWZ242H2udTMNpvZJjN7MrAxRYLbmr01PPneXq45bQyTMwZ7HUciVK+XfDOzKOBBYC5QDKw2s+XOuc099skBvgec5pyrMbP0vgosEmw6On3ctGwDw5Pi+ObcXK/jSATzZ4Q+CyhwzhU659qAp4EFB+1zPfCgc64GwDlXHtiYIsHrkX/vZmtpPbfPn0SiLosrHvKn0DOAoh7Pi7u39ZQL5JrZO2a20szmHeobmdkiM8s3s/yKioqjSywSREoONPPAG9s5Z0K67kAkngvUQdFoIAc4C7gc+J2Z/cdFn51zDznn8pxzeWlpaQH60SLeuWP5JnzOcfv8EzDTgVDxlj+FXgJk9Xie2b2tp2JguXOu3Tm3C9hOV8GLhK03Npfx+uYyvn5OLllD472OI+JXoa8GcsxsjJnFAJcByw/a5wW6RueYWSpdUzCFAcwpElSa2jq4ffkmcocnct0ZY7yOIwL4UejOuQ5gMfAasAVY6pzbZGZ3mtn87t1eA6rMbDPwNvBt51xVX4UW8drP3txByYFm7r5oCgN10woJEn4dknfOrQBWHLTtth6PHfCt7g+RsLa1tI6H/7WLz+dlcVL2UK/jiHxEQwuRI+DzOW5atoHBgwZy4/kTvI4j8jEqdJEj8Ex+EWv2HuCmCyYyJCHG6zgiH6NCF/FTZUMrP3xlK7PHDOXiGQefiiHiPRW6iJ/ueXkLTW0d3H3RZK05l6CkQhfxw793VrJsbQlfmnM849KTvI4jckgqdJFetHZ0cssLGxk1NJ7FZ4/zOo7IYelKQiK9+O3fCymsaOSRa04ibmCU13FEDksjdJFPsLuykV++XcBnph7HWeN1VWgJbip0kcNwznHrixuJjRrAbRdO8jqOSK9U6CKH8Zf1+/nnjkr+97zxDE+O8zqOSK9U6CKHUNvczl0vbWZKxmCuPHm013FE/KKDoiKH8OPXt1HV0MofvnCSbvgsIUMjdJGDrCs6wOMr9/A/p2QzJVM3fJbQoUIX6aGj08fNz28gLTGWJZ/WDZ8ltKjQRXp47N09bNpXx/f/6wSS4gZ6HUfkiKjQRbrtr23mx69v46zxaVwwRTd8ltCjQhfpdudfNtPhc9w5XxffktCkQhcB3tpaxisbS/naOTmMGqYbPktoUqFLxGtu6+S2FzcxLj2R688Y63UckaOmdegS8X7+1g6Ka5p5ZtHJxERrjCOhS7+9EtG2l9Xzu38UsnBmJrPHDvM6jsgxUaFLxPL5HDc/v4HEuGhuumCi13FEjpkKXSLWs+8Xs3p3DTedP5GhuuGzhAEVukSk6sY27nllCydlD2HhzEyv44gEhApdItI9K7bQ0NLB3RdNYYAuviVhQoUuEefdnVU8+34x188ZS+5w3fBZwocKXSJKa0cnN7+wgayhg/ja2TlexxEJKK1Dl4jy67/tpLCikUe/OItBMbrhs4QXjdAlYuysaOBXb+9k/rSRnJmb5nUckYBToUtEcK5rzXncwAHccqHWnEt4UqFLRHhuTQkrC6u58fyJpCfphs8SnvwqdDObZ2bbzKzAzG78hP0uNjNnZnmBiyhybKob27j75c3MHD2Ey07K8jqOSJ/ptdDNLAp4EDgfmARcbmaTDrFfEvB1YFWgQ4oci7tf3kJ9Swf3aM25hDl/RuizgALnXKFzrg14GlhwiP3uAu4FWgKYT+SY/HtnJc+tKWbRnLGMH6E15xLe/Cn0DKCox/Pi7m0fMbMZQJZz7uVP+kZmtsjM8s0sv6Ki4ojDihyJlvZObnl+I6OGxvNVrTmXCHDMB0XNbADwALCkt32dcw855/Kcc3lpaVo2Jn3rV28XUFjZyA8+O1lrziUi+FPoJUDPI0mZ3ds+lARMBv5mZruBk4HlOjAqXtqyv45f/W0nF03PYI7WnEuE8KfQVwM5ZjbGzGKAy4DlH77onKt1zqU657Kdc9nASmC+cy6/TxKL9KKj08d3nl1PSvxAbrvwP47fi4StXgvdOdcBLAZeA7YAS51zm8zsTjOb39cBRY7U7/65iw0ltdwxfzJDdJ1ziSB+XcvFObcCWHHQttsOs+9Zxx5L5OjsrGjgJ29u57wThnPBlBFexxHpVzpTVMKGz+e48bn1xEUP4K4FkzHTmnOJLCp0CRuPr9zD6t013HrhJNKTdXq/RB4VuoSFouom7n11K3Ny03RLOYlYKnQJec45bnp+Awbcc5GmWiRyqdAl5D353l7+uaOS754/gcwh8V7HEfGMCl1C2p6qRu5+eQunj0vlytmjvY4j4ikVuoSsTp9jydIPiBpg3Ldwqq6kKBFP9xSVkPW7fxaSv6eGBy6dxsiUQV7HEfGcRugSkraW1vHA69uZd8IILpqe0fsXiEQAFbqEnLYOH9985gOSB0Vzt1a1iHxEUy4Scn7+1x1s2V/HQ1fNZFhirNdxRIKGRugSUlYWVvGrvxWwcGYmnz5B12oR6UmFLiGjprGNbz6zjlFD47l9/glexxEJOppykZDgnOO7z62nsqGVZV8+jcRY/eqKHEwjdAkJf1q1l9c3l/Gd8yYwJXOw13FEgpIKXYLe1tI67nppM3Ny07j29DFexxEJWip0CWrNbZ187am1JMdF8+NLpulsUJFPoIlICWq3L9/E9rIGHv3iLNKStERR5JNohC5Ba+nqIp7JL+KGs47nzNw0r+OIBD0VugSljSW13PriRk4bN4wlnx7vdRyRkKBCl6BT29TODU+sYUh8DD+7bDpRmjcX8Yvm0CWo+HyOJX9ex74DzTzzpVNI1an9In7TCF2Cyq//vpM3t5Rzy2cmMnP0EK/jiIQUFboEjTc2l/Gj17cxf9pIvnBqttdxREKOCl2CwtbSOr7x9FqmZAzmvoVTdUlckaOgQhfPVTW0ct2j+STERvPQVXnEDYzyOpJISNJBUfFUW4ePLz+xhvL6VpZ+6RRGDI7zOpJIyNIIXTzjnOPWFzby3q5q7l84lROzUryOJBLSVOjimV+8VcAz+UV89exxLDhR9wUVOVYqdPHE0vwiHnhjO5+bkcG35uZ6HUckLKjQpd/9bVs531u2gTNyUvnh57SiRSRQ/Cp0M5tnZtvMrMDMbjzE698ys81mtt7M/mpmowMfVcLBhuJabnhiDeOHJ/HrK2cSE60xhUig9PqvycyigAeB84FJwOVmNumg3dYCec65qcCzwH2BDiqhb0dZPV/443sMiY/hkWtO0m3kRALMn+HRLKDAOVfonGsDngYW9NzBOfe2c66p++lKIDOwMSXU7a5s5L8fXkXUAOOJ62aTnqzliSKB5k+hZwBFPZ4Xd287nGuBVw71gpktMrN8M8uvqKjwP6WEtJIDzfz3w6to7/TxxHWzyU5N8DqSSFgK6ASmmV0J5AH3H+p159xDzrk851xeWppuWBAJyutbuPLhVdS1tPP4tbPJHZ7kdSSRsOXPJGYJkNXjeWb3to8xs3OBm4EznXOtgYknoay0toUrHl5JWV0Lj187m8kZg72OJBLW/BmhrwZyzGyMmcUAlwHLe+5gZtOB3wLznXPlgY8poaa4polLf/su5XWtPHLNLF0KV6Qf9DpCd851mNli4DUgCviDc26Tmd0J5DvnltM1xZII/Ll7TfFe59z8PswtQezDA6D1Le08fu0spo9SmYv0B7/WjTnnVgArDtp2W4/H5wY4l4SogvJ6rvhd1wHQJ68/WdMsIv1IC4ElYPJ3V3PdY/lEDxjA04tOYfwIHQAV6U86TU8C4tWN+7ni4VUMiY9h2ZdPVZmLeEAjdDlmj7yzizte2sz0rBQe/sJJDE2I8TqSSERSoctR6+j0cc+KrfzhnV3MnTScn182nUExutuQiFdU6HJUahrbWPzUGt4pqOLqU7O59cJJRA3QVRNFvKRClyO2tbSO6x/Lp6y2lfsWTuXSvKzev0hE+pwKXY7I8g/2ceNz60mMjebpL53MDK0xFwkaKnTxS3NbJ3f8ZRNPry5i5ugh/Oq/ZzBcV0wUCSoqdOnVttJ6Fj+5hoKKBm4463i+OTeXgVFa8SoSbFToclg+n+PxlXu4Z8UWkuKieeyLszgjR1fJFAlWKnQ5pL1VTXznuQ9YWVjNmblp3H/JVNKTNMUiEsxU6PIxPp/jT6v28MNXthJlxr0XT+HSvCzdyFkkBKjQ5SMbS2q57cWNrNl7gDm5afzwc1MYmTLI61gi4icVulDb3M4Dr2/j8ZV7GBIfw48umcbFMzI0KhcJMSr0CNbR6WNpfjEPvLGN6sY2rjp5NN/69HgGDxrodTQROQoq9AjknOO1TaXc9+o2CisbyRs9hEeumaVrl4uEOBV6BHHO8W5hFfe/to21ew8wLj2R3/1PHudOTNf0ikgYUKFHAOccb28r55dvFbBm7wGGJ8dy78VTuHhGJtE6QUgkbKjQw1h7p49XN5by67/tZPP+OjJSBnHXghO4JC+LuIG6zK1IuFGhh6HKhlaeWrWXJ1btpbSuhbGpCdy/cCqfnZ6hU/ZFwpgKPUw458jfU8NTq/by0vr9tHX6OCMnlR98djKfmpCua5WLRAAVeogrqm5i2ZoSlq0tZk9VEwkxUVw+K4urTslmXHqi1/FEpB+p0ENQeX0Lr28q46X1+1hZWA3AKWOH8bWzc5g3eQQJsfprFYlE+pcfIoprmnh1YymvbSolf08NzsHY1ASWzM3lohkZZA6J9zqiiHhMhR6kWto7WbWrmn9sr+Af2yvYUd4AwIQRSXzjnFzmTR5B7vBErR8XkY+o0INES3snG0pqeW9XNSsLq3hvVzWtHT5iogcwe8xQLs3LYu6k4WSnJngdVUSClArdIxX1rWwoOcDq3TXk767mg6Ja2jp9AIxLT+SK2aM4MzeN2WOGMShGa8ZFpHcq9D7mnGN/bQub9tWxsaS262NfLWV1rQBEDzCmZA7m6tOyyRs9hLzsoQxNiPE4tYiEIhV6gHT6HMU1Tewoa6CgooGC8q6PneUN1Ld2ADDA4Pi0RE47PpUTMgYzeWQyUzNTNAIXkYBQofvJOUd1YxtFNc0UVTdRVNNEUXUzxTVNFFU3UXKgmfZO99H+aUmx5KQnctGMDMalJ3LCyMFMPC6J+Bj9kYtI34j4dmnr8HGgqY2apnbK61soq2ulvL6F8u7PZXWtlNW1UF7fSluH72NfOzQhhqwhgzghYzDzJh/H2NQEjk9PZFx6oq4pLiL9zq9CN7N5wM+AKOBh59wPD3o9FngMmAlUAZ93zu0ObNRD6/Q5Gts6aGjpoKG1+6Olg8bWDupbuz4faGr/qLRrmto40P25prGNxrbOQ37fpNho0pNjSU+KI2/0ENKT4xiRHEfW0Hiyhg4ic0g8iTqBR0SCSK+NZGZRwIPAXKAYWG1my51zm3vsdi1Q45wbZ2aXAfcCn++LwM+s3stv/174UVk3HaaQD5YcF82QhBhS4mMYlhjDuPREUuIHMiQ+hiHxA0mJjyE9KZbhyXGkJ8dqakREQo4/rTULKHDOFQKY2dPAAqBnoS8Abu9+/CzwSzMz55wjwIYmxHJCxmASY6NIjI0mITaaxA8/4rqeJ334OKZre1JctK77LSJhz59CzwCKejwvBmYfbh/nXIeZ1QLDgMqeO5nZImARwKhRo44q8NxJw5k7afhRfa2ISDjr12Grc+4h51yecy4vLS2tP3+0iEjY86fQS4CsHs8zu7cdch8ziwYG03VwVERE+ok/hb4ayDGzMWYWA1wGLD9on+XAF7ofLwTe6ov5cxERObxe59C758QXA6/RtWzxD865TWZ2J5DvnFsO/B543MwKgGq6Sl9ERPqRX2vznHMrgBUHbbutx+MW4JLARhMRkSOhtXwiImFChS4iEiZU6CIiYcK8WoxiZhXAHk9++LFJ5aATpiKA3nNkiLT3HKrvd7Rz7pAn8nhW6KHKzPKdc3le5+hPes+RIdLeczi+X025iIiECRW6iEiYUKEfuYe8DuABvefIEGnvOezer+bQRUTChEboIiJhQoUuIhImVOjHwMyWmJkzs1Svs/QlM7vfzLaa2Xoze97MUrzO1FfMbJ6ZbTOzAjO70es8fc3MsszsbTPbbGabzOzrXmfqL2YWZWZrzewlr7MEigr9KJlZFvBpYK/XWfrBG8Bk59xUYDvwPY/z9Ike9889H5gEXG5mk7xN1ec6gCXOuUnAycBXIuA9f+jrwBavQwSSCv3o/QT4DhD2R5Wdc6875zq6n66k6yYn4eij++c659qAD++fG7acc/udc2u6H9fTVXAZ3qbqe2aWCXwGeNjrLIGkQj8KZrYAKHHOfeB1Fg98EXjF6xB95FD3zw37cvuQmWUD04FV3ibpFz+la0Dm8zpIIPl1PfRIZGZvAiMO8dLNwE10TbeEjU96v865F7v3uZmu/6I/0Z/ZpO+ZWSLwHPAN51yd13n6kpldCJQ75943s7O8zhNIKvTDcM6de6jtZjYFGAN8YGbQNf2wxsxmOedK+zFiQB3u/X7IzK4GLgTOCePbC/pz/9ywY2YD6SrzJ5xzy7zO0w9OA+ab2QVAHJBsZn9yzl3pca5jphOLjpGZ7QbynHOheNU2v5jZPOAB4EznXIXXefpK9w3OtwPn0FXkq4ErnHObPA3Wh6xrVPIoUO2c+4bXefpb9wj9f51zF3qdJRA0hy7++CWQBLxhZuvM7DdeB+oL3Qd+P7x/7hZgaTiXebfTgKuAs7v/btd1j1wlBGmELiISJjRCFxEJEyp0EZEwoUIXEQkTKnQRkTChQhcRCRMqdBGRMKFCFxEJE/8fdQob6HXKMBQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "a = np.arange(-5, 5, 0.01)\n",
    "plt.plot(a, sigmoid(a));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEWCAYAAACHVDePAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAc3klEQVR4nO3dd9hU9ZnG8e8tkoiKHRtgUJKggJVXLIklVsya6G7UWDdNWRV3oyZRiWmmurHErDG4rDGJEdA1Yoo1GHWjxka1IHakKEVRARGkPPvHOaMDeZEzLzNnyrk/1zUXU86c3zPA9cxvfufMPYoIzMysta1T7wLMzKz23OzNzArAzd7MrADc7M3MCsDN3sysANzszcwKwM3eWp6kkyT9pUb7vlrSt2uwX0n6taQ3JD1a7f2vYew7JH0hzzGt9uTz7K2cpKnAVsByYCFwJ3BWRCzM8NwDgesjokc7j92XPnZNlu07SlIv4CWgc0Qsq9Z+031/ETg1Ij5Zzf2uZqz9gFFAn4h4u4bjfA/4aEScXKsxrDF4Zm/t+UxEbAjsBuwODK1zPUX0EWBqLRu9FYubva1WRMwC7iJp+gBI+rCkSyVNkzQ7XcboUovxJd0kaZaktyT9TVK/sse6SLpM0svp4w+kdfwt3eRNSQsl7SPpi5IeSJ83TNKlq4zzR0nnptcvkPSCpAWSJkv65/T+nYCrgX3S/b6Z3v8bST8s29dpkp6XNE/SnyRtW/ZYSDpd0nOS3pR0lSS187q/AlxTNtZF5a9hlf19tKyOqyTdltb+iKTeZdv2kzQmrWu2pG9KGgR8E/h8Os6kdNv7JJ2aXl9H0rfSv+c5kq6TtHH6WK+0hi+k/x9ek3RhJf/Glh83e1stST2AI4Dny+6+GPg4yRvAR4HuwHdqVMIdwMeALYHxwIiyxy4FBgD7ApsB5wErgP3TxzeJiA0j4qFV9jmKpLkJQNKmwGHADenjLwD7ARsDFwHXS9omIp4GTgceSve7yarFSjoI+AlwHLAN8HLZfkuOBPYEdkm3O3zV/UTEr1YZ67ur/Rta2fFpzZuS/Jv9KK2rK3A3yZLctiT/bn+NiDuBHwM3puPs2s4+v5hePgXsAGwI/GKVbT4J9AEOBr6TvjFag3Gzt/b8QdICYDowB/guJAcNgcHAORExLyIWkDSL42tRRERcGxELImIJ8D1gV0kbS1oH+DLw1YiYGRHLI+Lv6XZrcj8QJA0d4BiSpvpKOuZNEfFKRKyIiBuB54CBGUs+Cbg2IsantQwlmZ33Ktvm4oh4MyKmAfdS9qmpCm6JiEfTYxUjyvZ9JDArIi6LiMXp3+kjGfd5EnB5RLyYHrcZChwvad2ybS6KiHciYhIwCWjvTcPqzM3e2nN0RHQFDgR2BLZI7+8GrA+MS5ch3iSZLXbLsM9lQOdV7usMLG1vY0mdJF2cLqnMB6amD22RXtYjmYVXJJIzEm4ATkjvOpGyTwyS/lXSxLLX15/3X/+abEsymy+NtRB4neTTT8mssuuLSGbK1bK6ffekA39XqZVeU3p9XZKD+Gsa1xqIm72tVkT8H/AbkiUTgNeAd4B+EbFJetk4PZi7JtOAXqvctz0rN5JyJwJHAYeQLKmUnqu0jsVA73ael+X0slHAMZI+AuwF3AyQ3v4f4Cxg83Sp5sl0zCz7foXkwCrp/jYANgdmZqhpTd4meaMt7XvrCp47nWQJpj0VvSZgO5I37tkVjG8NwM3e1uQK4FBJu0bECpJm+DNJWwJI6i5ppXVnSeutchFwI/AlSQOV+DhwDv+4pl3SFVhCMjNen2S5CIC0jmuByyVtm34K2EfSh4G5JGv3q2tuRMQEkjeMa4C7IuLN9KENSJrf3PR1fIlkZl8yG+gh6UOr2fWo9DXultbyY+CRiJi6uloqMAnol+57PZJlraxuBbaRdLaSA+xdJe2VPjYb6JUujbVnFHCOpO0lbcj7a/xVPa3Vas/N3j5QRMwFruP9g7Dnkxz8ezhdXrmb5OBcSXeS2X/5pXdE3AVcAPwaeAu4HfgtMHw1Q19HMuufCUwGHl7l8a8DTwCPAfOA/wTWiYhFJAcmH0yXYvZezf5HknxqGFn2WicDlwEPkTTBnYEHy55zD/AUMEvSa6vuMCLuBr5N8knhVZJPHlU5nhERzwLfJ/n7fg544IOfsdJzFwCHAp8hWXJ5juSAK8BN6Z+vSxrfztOvBX5HcpbTSySfqP69Ay/B6sxfqjIzKwDP7M3MCsDN3sysANzszcwKwM3ezKwA1l3zJvWxxRZbRK9evepdhplZ0xg3btxrEdHulxwbttn36tWLsWPH1rsMM7OmIWl1X1L0Mo6ZWRG42ZuZFYCbvZlZAbjZm5kVgJu9mVkB5HY2jpIfsl5A8kPWyyKiLa+xzcyKLu9TLz8VEf+QFmhmZrXlZRwzswbx6EvzuOb+F6lFGnGezT6Av0gaJ2lwextIGixprKSxc+fOzbE0M7P6mrNgMUNGjmfEI9N4Z+nyqu8/z2b/yYjYAzgCGCJp/1U3iIjhEdEWEW3dumX5WVMzs+a3bPkK/mPUBBYsXsqwk/dg/Q9Vf4U9t2YfETPTP+cAtwAD8xrbzKyRXTbmWR5+cR4//ued2XHrjWoyRi7NXtIGkrqWrgOHkfyQs5lZoY2ZPJth973ACQO341/26FGzcfI6G2cr4Jbkd6dZFxgZEXfmNLaZWUOa9voizv3fifTvvhHf/Uzfmo6VS7OPiBeBXfMYy8ysGSxeupwzRoxjHYlhJw1gvc6dajpew0Ycm5m1su/96SmeemU+136xjZ6brV/z8XyevZlZzm4aO50bHpvOkE/15qAdt8plTDd7M7McTX5lPt/6w5Ps23tzzj20T27jutmbmeVk/uKlnDliHJus35n/OmF3Oq2j3Mb2mr2ZWQ4igm/cNInpb7zDjYP3ZosNP5zr+J7Zm5nl4Jr7X+Kup2Yz9Igdaeu1We7ju9mbmdXYoy/N4+I7p3BE/635yie3r0sNbvZmZjU0Z8Fizho5nu02W5+fHrML6ZdLc+dmb2ZWI6WAs/lpwFnX9TrXrRYfoDUzq5FSwNllx+5as4CzrDyzNzOrgfKAs88NqF3AWVZu9mZmVZZnwFlWbvZmZlWUd8BZVl6zNzOroov+nASc/eoL+QScZeWZvZlZlfx+3AxGPTqdMw/szcE75RNwlpWbvZlZFTz96nwuvOUJ9tlhc8499OP1LucfuNmbma2l+YuXcsb149i4SxJwtm6nxmutXrM3M1sLEcF5Nz3O9Dfe4YbBe9Ota74BZ1k13tuPmVkTueb+l7jzqVkMPWJH9qxDwFlWbvZmZh3UCAFnWbnZm5l1QKMEnGXlZm9mVqFGCjjLygdozcwqdHkacHZpAwScZeWZvZlZBe6ePJtf3vcCJwzsyTENEHCWlZu9mVlGKwec9at3ORVxszczy2Dx0uWcOXIcQEMFnGXlNXszswwu+vNTPDmz8QLOsvLM3sxsDRo54CwrN3szsw/Q6AFnWbnZm5mtRjMEnGWVa+WSOkmaIOnWPMc1M6tUecDZVSft0bABZ1nl/Tb1VeDpnMc0M6vYrx5IAs4uGNTYAWdZ5dbsJfUA/gm4Jq8xzcw64rGp8/jJHVMY1G9rTt2vsQPOsspzZn8FcB6wYnUbSBosaayksXPnzs2vMjOz1NwFSxgyYjw9N+3CT49t/ICzrHJp9pKOBOZExLgP2i4ihkdEW0S0devWLY/SzMzes3LA2QA2aoKAs6zymtl/AvispKnADcBBkq7PaWwzs0wuH/MsD734Oj88emd22qY5As6yyqXZR8TQiOgREb2A44F7IuLkPMY2M8uiWQPOsmrek0bNzKqkmQPOsso9Gyci7gPuy3tcM7P2NHvAWVYOQjOzQisFnF3zr80ZcJaVl3HMrLBKAWdnHNibQ/o2Z8BZVm72ZlZIU2bN51t/SALOvtbEAWdZudmbWeEkAWfj2Wi95g84y8pr9mZWKKWAs2nzFjHqtL2bPuAsq9Z/OzMzK1MecDZw++YPOMvKzd7MCqMVA86ycrM3s0Jo1YCzrLxmb2YtrxRw9tY7S/ntlwe2VMBZVm72ZtbyfnZ3EnB2yTG7tFzAWVZexjGzlvbXp2dz1b0vcPyePTm2rWe9y6kbN3sza1nT5y3inBsn0m/bjfjeZ1sz4CwrN3sza0mLly7njBGtH3CWldfszawlXfTnye8FnG23eesGnGXlmb2ZtZybx81g1KPTChFwlpWbvZm1lCmz5nNhgQLOsnKzN7OWUcSAs6y8Zm9mLaGoAWdZ+W3PzFpCKeDs/EF9ChVwlpWbvZk1vbFT53HxHVM4vN9WnLbfDvUupyG52ZtZU3tt4RKGjBxPj027cMmxuxYu4CwrN3sza1rLVwT/MWoCby5ayi9PGlDIgLOsfIDWzJrW5WOe4e8vJAFnfbctZsBZVp7Zm1lTcsBZZdzszazpOOCscm72ZtZUSgFngQPOKuE1ezNrKqWAs/9xwFlFPLM3s6ZRCjg7/YDeHOqAs4q42ZtZUygFnO29w2Z8/TAHnFXKzd7MGt4CB5ytNa/Zm1lDiwjO+/37AWdbdl2v3iU1pVzeHiWtJ+lRSZMkPSXpojzGNbPm96sHXuKOJx1wtrbymtkvAQ6KiIWSOgMPSLojIh7OaXwza0IOOKueXJp9RASwML3ZOb1EHmObWXNywFl15XaUQ1InSROBOcCYiHiknW0GSxoraezcuXPzKs3MGowDzqovt2YfEcsjYjegBzBQUv92thkeEW0R0datW7e8SjOzBlMKOPvB0f0dcFYlFTd7SRtI6vD3kyPiTeBeYFBH92FmreueKUnA2efbenKcA86qZo3NXtI6kk6UdJukOcAU4FVJkyVdIumjGfbRTdIm6fUuwKHpfszM3pMEnE2i7zYbcdFRDjirpiwz+3uB3sBQYOuI6BkRWwKfBB4G/lPSyWvYxzbAvZIeBx4jWbO/dS3qNrMWs3jpcs4cMZ4VEVx9sgPOqi3L2TiHRMTSVe+MiHnAzcDN6emUqxURjwO7d6xEMyuC7986mSdmvuWAsxpZ48y+1Ogl/VyrOfepvTcDM7OsRo+fwchHHHBWS5UcoF0A/EnSBgCSDpf0YG3KMrOimDJrPt+8xQFntZb5S1UR8S1JJwL3SXqX5EtSF9SsMjNreaWAs64OOKu5zM1e0sHAacDbJAdcvxwRz9SqMDNrbeUBZyNP3csBZzVWydvohcC3I+JA4BjgRkkH1aQqM2t5pYCz8w7vw147bF7vclpeJcs4B5Vdf0LSESRn4+xbi8LMrHWVAs4O67sVg/d3wFkesnypanVn4LwKHPxB25iZraoUcNbdAWe5yvSlKkn/Lmm78jslfQjYR9JvgS/UpDozaynLVwRfvSEJOBt20gA27uKAs7xkWcYZBHwZGCVpB+ANoAvJG8VfgCsiYkLtSjSzVvGzMc/y4POv89NjdnHAWc6yNHtFxC+BX6bflN0CeCcNNDMzy+SeKbP5xb3PO+CsTrI0+1fSALTHyy5/q2lVZtZSHHBWf1niEjYlWcq5Pr3rZOApSaMkbVzL4sys+ZUHnA07eQ8HnNVJplMvI+Il4CXgj/De2TcXAlcAX6pZdWbW9EoBZ8NPGcBHNt+g3uUUVod+gzb9TdkfSnq6yvWYWQspBZz92wE7cFi/retdTqFlOc/+XEmHSNpylfs/DPj7zWbWrlLA2V7bb8Y3DutT73IKL8vMfivgXGBnSesCTwAvAHuSfIPWzGwlCxYv5cw04OzKEx1w1gjW2Owj4vzS9fSnBXcG+gCjI2JMDWszsyYUEZx/8+O87ICzhlLRmn16bv396QVJD0bEJ2pRmJk1p2sfnMrtT8xi6BE7OuCsgaztZ6ttq1KFmbWEsVPn8ZPbn3bAWQNa48xe0pUk6/RPAE9GxIKyh6NWhZlZc3HAWWPLsozzBMk6/UlAf0nzeb/5d61hbWbWJMoDzkafuacDzhpQlgO0w8tvS+pB0vx3Ae6qUV1m1kTeCzj73C7029ZfrG9EFX+pKiJmADOAO6pfjpk1m1LA2XFtPThuTwecNarMB2glfUzStZKuqmVBZtY8ygPOvn9U/3qXYx+gkrNxfgfcBOwHIKm/pOtqUpWZNbwly5YzZKQDzppFJc1+nYi4A1gOEBFPAn4rNyuo7/95Mo/PeIvLjt3VAWdNoJJm/4qk7UlPt0yTL7vUpCoza2i3TJjBCAecNZVKDtCeDVwDbC3pSyQZ90/WpCoza1jPzFrA0NEOOGs2mZt9REyVNAg4GtgVuA+4tkZ1mVkDWrB4KWdcP84BZ00oS8TxKZLmSpoBnBgRvwduA7oDf691gWbWGMoDzq48YXcHnDWZLG/L3wE+DewO7CBpDMlZOZ1JlnbMrABKAWffOLwPezvgrOlkWcZZGBGPAUi6CJgNfDxNwMxEUk/gOpJs/ACGR8TPO1CvmdXBuJeTgLND+27FvzngrCllafZbSxoMPJNeZlTS6FPLgK9FxHhJXYFxksZExOQK92NmOXtt4RKGjJhA9027cKkDzppWlmb/Xd4PQtsZ6CrpbmACMCEiRq5pBxHxKvBqen1B+tu13QE3e7MGVgo4e2PRu4w+c18HnDWxtQ1COwJYY7Nf5fm9SNb/H2nnscHAYIDtttuukt2aWQ1ccbcDzlpFrkFokjYk+d3asyNifjv7Hg4MB2hra3NWvlkd3TtlDlfe44CzVpHbSbKSOpM0+hERMTqvcc2sctPnLeLsGyc64KyF5NLs02iFXwFPR8TleYxpZh3jgLPWlNfM/hPAKcBBkiaml0/nNLaZVaAUcHapA85aSsVr9h0REQ8APl/LrMG9F3C2/w4c7oCzluJgCzMDkoCzb45+koHbb8Y3DnfAWatxszczFi5ZxhkjxrHBh9flFyc44KwV5bKMY2aNKyI4//eP8/Lrixhx6l5suZEDzlqR377NCu7XD07ltidedcBZi3OzNyuwcS/P48cOOCsEN3uzgioFnG27iQPOisBr9mYFVAo4m7foXUaf4YCzIvDM3qyASgFnPziqH/27O+CsCNzszQqmFHB27IAefH5Pp8sWhZu9WYHMeCMJONtpm434wdEOOCsSN3uzgliybDlnjhjPihXBsJMccFY0PkBrVhA/uDUJOPvvUwbQawsHnBWNZ/ZmBfCHCTO5/mEHnBWZm71Zi3t29gKGjn7CAWcF52Zv1sIWLlnG6dc74Mzc7M1aVingbOprb3PlCbs74Kzg3OzNWtT7AWc7sk9vB5wVnZu9WQsqBZwdstNWnH6AA87Mzd6s5bxeFnB22XEOOLOEz7M3ayFJwNlEB5zZP/DM3qyF/PzuZ3ng+dcccGb/wM3erEXc+8wc/ssBZ7YabvZmLWDGG4s458aJ7Lh1VwecWbvc7M2aXCngbPny4OqTBzjgzNrlA7RmTa4UcHb1yQ44s9XzzN6siZUCzgbvvwOD+jvgzFbPzd6sSb0XcNZrM85zwJmtgZu9WRNaKeDsRAec2Zr5f4hZk4kIzr/ZAWdWGTd7sybzm79P5bbHHXBmlcml2Uu6VtIcSU/mMZ5Zqxr38hv86DYHnFnl8prZ/wYYlNNYZi3p9YVLOGvkeLbZZD0HnFnFcmn2EfE3YF4eY5m1olLA2etvv8uwkwY44Mwq1lBr9pIGSxoraezcuXPrXY5ZwygFnH3/sw44s45pqGYfEcMjoi0i2rp161bvcswaQing7JgBPfj8nj3rXY41qYZq9ma2spUCzo7q73V66zA3e7MGtWTZcoaUBZx1+ZADzqzj8jr1chTwENBH0gxJX8ljXLNm9sNbn2bSjLe45NhdHXBmay2X1MuIOCGPccxaxR8nzuR3D7/sgDOrGi/jmDWYZ2cv4IKbHXBm1eVmb9ZAHHBmteL/SWYNwgFnVktu9mYNohRw9vXD+zjgzKrOzd6sAbwfcLYlp+/fu97lWAtyszers5UCzo7djXXW8RenrPr8g+NmdbR8RXD2jUnA2egz9mXj9R1wZrXhmb1ZHf38r89x/3MOOLPac7M3q5P7npnDlfc854Azy4WbvVkdzHhjEWffOJE+WzngzPLhZm+Ws/KAs2EOOLOc+ACtWc5KAWdXn7wH2zvgzHLimb1ZjkoBZ6fttz2D+m9T73KsQNzszXJSCjjbs9emnDdox3qXYwXjZm+Wg5UDzvagswPOLGf+H2dWYxHBBWUBZ1s54MzqwM3erMZ++/ep3OqAM6szN3uzGho/7Q1+dLsDzqz+3OzNauT1hUsYMmI8W2/sgDOrP59nb1YDDjizRuOZvVkNlALOLnLAmTUIN3uzKisFnH1ujx4c74AzaxBu9mZVVB5w9sOjHXBmjcPN3qxKlixbzpCRExxwZg3JB2jNquRHtz3NpOlvOuDMGpJn9mZV8MeJM7nuIQecWeNyszdbS8854MyagJu92Vp4P+CskwPOrKF5zd6sg0oBZy+99jbXn7qXA86soXkaYtZBpYCzrx3Wh317b1Hvcsw+kJu9WQeUAs4O3nFLzjjAAWfW+HJr9pIGSXpG0vOSLshrXLNqKw84u/w4B5xZc8il2UvqBFwFHAH0BU6Q1DePsc2qqTzgbNhJAxxwZk0jrwO0A4HnI+JFAEk3AEcBk6s90GeufIDFS5dXe7dmACxZtoJp8xbxk3/Z2QFn1lTyavbdgellt2cAe626kaTBwGCA7bbbrkMD9e62Ae8uX9Gh55plccreH3HAmTWdhjr1MiKGA8MB2traoiP7uOL43atak5lZK8jrAO1MoHwq1CO9z8zMcpBXs38M+Jik7SV9CDge+FNOY5uZFV4uyzgRsUzSWcBdQCfg2oh4Ko+xzcwsxzX7iLgduD2v8czM7H3+Bq2ZWQG42ZuZFYCbvZlZAbjZm5kVgCI69N2lmpM0F3i53nVUaAvgtXoXkTO/5mLwa24OH4mIbu090LDNvhlJGhsRbfWuI09+zcXg19z8vIxjZlYAbvZmZgXgZl9dw+tdQB34NReDX3OT85q9mVkBeGZvZlYAbvZmZgXgZl8jkr4mKSRtUe9aak3SJZKmSHpc0i2SNql3TbUiaZCkZyQ9L+mCetdTa5J6SrpX0mRJT0n6ar1ryoOkTpImSLq13rVUi5t9DUjqCRwGTKt3LTkZA/SPiF2AZ4Ghda6nJiR1Aq4CjgD6AidI6lvfqmpuGfC1iOgL7A0MKcBrBvgq8HS9i6gmN/va+BlwHlCIo98R8ZeIWJbefJjkl8ha0UDg+Yh4MSLeBW4AjqpzTTUVEa9GxPj0+gKSBti9vlXVlqQewD8B19S7lmpys68ySUcBMyNiUr1rqZMvA3fUu4ga6Q5ML7s9gxZvfOUk9QJ2Bx6pbyU1dwXJZG1FvQuppob6wfFmIeluYOt2HroQ+CbJEk5L+aDXHBF/TLe5kORj/4g8a7Pak7QhcDNwdkTMr3c9tSLpSGBORIyTdGC966kmN/sOiIhD2rtf0s7A9sAkSZAsZ4yXNDAiZuVYYtWt7jWXSPoicCRwcLTulzdmAj3LbvdI72tpkjqTNPoRETG63vXU2CeAz0r6NLAesJGk6yPi5DrXtdb8paoakjQVaIuIZkvOq4ikQcDlwAERMbfe9dSKpHVJDkAfTNLkHwNObOXfU1Yya/ktMC8izq53PXlKZ/Zfj4gj611LNXjN3qrhF0BXYIykiZKurndBtZAehD4LuIvkQOX/tnKjT30COAU4KP23nZjOeq3JeGZvZlYAntmbmRWAm72ZWQG42ZuZFYCbvZlZAbjZm5kVgJu9WQZp+uNLkjZLb2+a3u5V38rMsnGzN8sgIqYDw4CL07suBoZHxNS6FWVWAZ9nb5ZRGhswDrgWOA3YLSKW1rcqs2ycjWOWUUQslfQN4E7gMDd6ayZexjGrzBHAq0D/ehdiVgk3e7OMJO0GHEryi03nSNqmziWZZeZmb5ZBmv44jCTPfRpwCXBpfasyy87N3iyb04BpETEmvf1LYCdJB9SxJrPMfDaOmVkBeGZvZlYAbvZmZgXgZm9mVgBu9mZmBeBmb2ZWAG72ZmYF4GZvZlYA/w/zXU/jNe6+GQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def relu(x):\n",
    "    return np.array([el if el > 0 else 0 for el in x])\n",
    "\n",
    "a = np.arange(-5, 5, 0.01)\n",
    "plt.plot(a, relu(a))\n",
    "plt.title(\"ReLU activation function\")\n",
    "plt.xlabel(\"X\")\n",
    "plt.ylabel(\"$ReLU(x)$\");\n",
    "# plt.savefig(IMG_FILEPATH + \"04_relu_activation.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAGFCAYAAADO2pt/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXxU1f3/8ddJIGwJWwJhCZsgEBJIgADKElABBReUSt2qUqUWW61av21tv9raVqs/raBUbUvV1rXql1bFXVRAUEDCGgiyJmwhIQGBsIRs5/fHnYlJSMgkmcnNJO/n45FHkpk7535yA3nPOffcc421FhEREXFPiNsFiIiINHUKYxEREZcpjEVERFymMBYREXGZwlhERMRlCmMRERGXKYxFRERcpjAWERFxmcJYGgRjzGZjzIRga7umjDEDjDHrjTF5xpif1eB1xhiz2xjTL5D11XS/xpgHjDHPlPn+EWPM3T62/bUxJs5ftVazr38ZYx6qj31Vsu+9xphhZ3nep2NWn8dL6l8ztwuQ4GOMyQCigSKgGEgDXgLmW2tLatOmtdYvf2Q8tc2y1n7q77b95JfAYmttYk1eZJ2l8noFpqQ67TcOWAxgjOkE3AT4+obhz8AfgO/Vpca6MsbcDHjfUDQDmgOnPN8XAdHW2tO1bLsD0A3YUsXzNTlmDeJ4SWCoZyy1dbm1NgLnD/WjwK+A52vaiDGmqb0h7AVsdrsIP4oDNnq+ngl8YK09VfXm5SwELjDGdAlEYb6y1r5orQ231oYDfwLe835vrW1f2yD2GAxknOWYzMT3Y9YgjpcEhsJY6sRae9RauxC4BrjZGBMPYIzpZoz5jzEmxxiTXnZI1hiTYYz5lTFmI3DCGNPM89hEz+MLyu7DGPOUMWae5+v7jDE7PcO8acaYq8ps9zLQE3jXGHPcGPPLMvvzpe2z1fwrY8x+z363GmMuqux4GGNijTFLjDFHPMPjV5R57nPgAuBpT339K3n9OcaY940xucaYY8aYRZ7HbzXGvO35OsQY81tjzCFjTKYx5jpjTIGnF+bd9mNjzF+NMd8aY7YZYwYZY+4yxuzxtD29zD6N5+fb7an7TWNMuyr2+2tjzEHPfq/F6dFt8jQ1BVha4ed5zPt6z/ePG2M+M8aEWWvzgTXAxVUcyyp/12V+r/9jjNlojDlqjHnDGNPS89xQY8xaz2vfAFpWto9KJAIbKuynszFmoTEm29Peu8aYtp7nfmyM+cAY84znuGYaYyaVefkQYIcxZp7n31XF58sds7ocLwly1lp96KNGH0AGMLGSx/cAt+O8yVsD/BYIA84BdgEXl3n9eqAH0Kpsmzg9x5NAhOfxUOAAcJ7n+xk4w34hOG8ATgBdz1abL22frWZgALAX6OZ5XW+gbyU/f3NgB/AbTxsXAnnAgDLbLMEZRq/q2C4HfuqprSUwxvP4POD3nq//AHwBdAfaA6uAvWXamAccAi7ytPMfYDdwl6fGu4C1ZbZ/yFNXV88+3wGeqGS/DwJfAl2Adp6vd5VpJwcYUeHniQSOAkOB2UAq0K5CrXOqOBa+/K6/9mzTEWcoeLbn2O8G7vH8vFcDhcBDPvzb3gVMr/BYP2AS0MKznxXALzzPPQsc9vw7CQF+B3xa5rV/9/wuqnq+3DGry/HSR3B/qGcs/pSJ88dqBNDJWvsHa22BtXYX8A/g2jLbzrPW7rUVhuestbuBtYC3F3QhcNJau9Lz/P9ZazOttSXW2jeA7cBIX4qrpu2z1VyM84d4kDGmubU2w1q7s5JdnAeEA4962vgceA+4zpf6PPriBGiotTbfWvul5/EEYINxzjHeDdxird1vrT0CfITzR5sy2z5irf3MWus9p59qrX3KWluI05NtBmCMiQbuBK631h6wTu9rAZBUyX7/B7jJWptlrT0KvF9hv+1x3nyUstYeAuYCLwK/BqZ6XuuV53ndGXz8Xc/zbHMYeBenZ3seTgg/aa0ttNYuAFZXto+yPL3d3jhvFMvWscNau8hae9qzn0VAB8/TQ3B+3x9bZ75EWoVmhwAPn+X5csesLsdLgpvCWPypO04voRfQzTPkecQYcwSntxhdZtu9Z2nnNb4LsOs93wNgjLnJOLORve3GA1E1qLGqtqus2Vq7AycAHwQOGmNeN8Z0q6Ttbjg91LKT2HbjHBdf3QBMAzKNMc8bYzp6Hh+CM3x6EbDRU5NXR8qH4hCcNwFegyr5/hvP1+NwgjqzzPNROCMGFfe7pcKbkGi+O18M8C0QUcnPtA7n3OmvrbUVf+8RwJFKXuPr7zqrzNcncd4MdQP2W2vL3h92d2X7qCABJ+zSK9QxwxjzpWd4/ghwH7DNGGM8P9e7ZTaPxxO4nufjq3reo7JjVqvjJcFNYSx+YYwZgRM6y3GCNt06k1+8HxHW2qllXnK2G2n/HzDBGBOD04t9zbOPXji91TuASGtte5xenvGx3Srbrq5ma+1r1tqxOKFtgf9XSduZQA9jTNn/Vz2B/dXU9F3x1n5urb0IJzATgJmen7sZzhBqFGX+GBtjQnHOO270fN8LZ5h2W5lmEynf2xtS5vtOOMOiZU0Dlley34Nl9tscuJLyYbwRKHce3BgzGPgrTk/vlkp+5FgqnKMt83NU97uuygGguycMvXr68LpEnDc6pf+GjDEX4vyu78YJee9xWI/Ti24GbC3TxlC+O7Z9PJ93VPE8VDhmtT1eEvwUxlInxpi2xpjLgNeBV6y1qTjn8fI8k4JaGWNCjTHxnsCulrU2B+cc5j9xAtJ7WUgbnCDM8ez7hzg9jbKycc731rTtKms2zrXBFxpjWgD5OJe9VHYJ1yqc3tkvjTHNjXNt8+WeY1MtY8x0Y8y5nhCJwBkKXY8Tyt6Q2AKMMcb08wyrzsMZ2vb2jBNwerolnjbb4ryBKBuaCXz3B301cL4xpq8xJtwY8wecHu8LFfa7FRhrjOnvmdz1V5yAK9sj/wAYX+bn6Y7TK5wN/AQYbMpc7+2ZbDUcZ9i3Il9+11VZgXNJ0s88v4fp+HYqo+KbFnCOwV6c49UB57h0xundDqHMsfYYynfH1vu8reJ5KHPM6ni8JMgpjKW23jXG5OH8ofpfYA7wQwDPecrLcP64pQO5wHM4k3589RrOpKvSIWprbRrwBM4f22ycobwvK7zuEeB+z9Dm/9Sg7bPV3ALn8q1cnGHRzjjn88qx1hbghO8Uz7bP4pxj/abitlUYizOzNg/nj/SjnvPOCXjC1Fr7GfBvnKHMFJzwOMl3w84JnNkL3mGtPQnOjGicUFvvaS8FeBhnRGMfTs/rQs/2Zfe7COdNRQpOgOfgvDHZXmZfLwFTPW9m2np+hjnW2oWe9h737MvrcmBJhSFyPPvz5XddKc/vYTrOZUOHcSZ//deHl1Y8dgCv4px/Powz1L8dSPPso+wIA8aYKJzJbd7Z5YMpE7yVPA/fHbN21OF4SfAz5d+0iUgwMcbMBi611l7udi0Axpg/AQettU/6sO0q4FZr7abqtm3MfD1mOl6Nm8JYJIgYY87DOSe6F2dS1as4C7CscrUwEamTprb6kUiwG4pzSVFznElaMxXEIsFPPWMRERGXaQKXiIiIyxTGIiIiLnPtnHFUVJTt3bu3W7sXERGpV2vWrMm11naq7DnXwrh3796kpKS4tXsREZF6ZYypcllWDVOLiIi4TGEsIiLiMoWxiIiIy7Toh4g0eoWFhezbt4/8/Hy3S5EmoGXLlsTExNC8eXOfX6MwFpFGb9++fURERNC7d2/K31lRxL+stRw6dIh9+/bRp0+f6l/goWFqEWn08vPziYyMVBBLwBljiIyMrPEojMJYRJoEBbHUl9r8W1MYi4iIuExhLCIi4rJqw9gY84Ix5qAxptIbWhvHPGPMDmPMRmPMMP+XKSIS/EJDQ0lMTCQuLo6EhASeeOIJSkpKatTG6NGja7XvI0eO8Oyzz/qlLV/NmzeP2NhYbrjhBp+2/+qrr/jtb38b0JrOto9Zs2bx3nvvcerUKcaPH09xcXGV7RQUFJCcnExRUZFf6qr2ForGmGTgOPCStTa+kuenAncCU4FRwFPW2lHV7TgpKclqOUwRqQ9btmwhNjbW7TIIDw/n+PHjABw8eJDrr7+eMWPG8Pvf/77a11prsdYSElK7Ac2MjAwuu+wyNm2qtF8VEAMHDuTTTz8lJiam3vZZF8OHD+edd97hnXfeoaioiLvuuuus2//+97+nX79+lb7ZqOzfnDFmjbU2qbK2qv2tWmu/AA6fZZNpOEFtrbUrgfbGmK7VtSsiLjiyB9K/gKP73a6kyevcuTPz58/n6aefxlrLK6+8wsiRI0lMTOTHP/4xxcXFZGRkMGDAAG666Sbi4+PZu3cv4eHhANx3330888wzpe09+OCD/PnPfwbgyiuvZPjw4cTFxTF//vzS7Xfu3EliYiK/+MUvAHxqq7K6KpozZw7x8fHEx8fz5JNPAjB79mx27drFlClTmDt37hmvefHFFxk+fDhDhgxh7NixAMyYMYNly5YBTpglJyczZMgQHn/8cfr161e6zR133MHYsWPp1asXy5cv58Ybb6R///7ceuutpe1/8803XHjhhSQmJjJx4kRyc3PP2Me2bdsYO3YsgwcP5uGHHyYrK4uYmBheffVVpk2bVtrWBRdcwKJFiwC4//77ufPOO0uP86uvvnr2X7SvvO+2zvYB9AY2VfHce8DYMt9/BiRVse1tQAqQ0rNnTysi9eRYlrWvXWvt79p+9/HGjdYez3W7snqRlpZW+vWDCzfZ7//tK79+PLhwk091tGnT5ozH2rVrZ5csWWIvu+wyW1BQYK219vbbb7cvvviiTU9Pt8YYu2LFijPaWLt2rU1OTi59PDY21u7Zs8daa+2hQ4estdaePHnSxsXF2dzcXJuenm7j4uIqraeqttLS0iqtq6yUlBQbHx9vjx8/bvPy8uygQYPs2rVrrbXW9urVy+bk5JzxMx87dszGxsba06dPW2ut/fbbb6211g4cONAeOXLEFhYW2qFDh5a2M3v2bDtt2jRrrbUDBgywTzzxhLXW2ocfftj279/fZmZm2sLCQhsdHW3z8/Ntfn6+HTRokF23bp211tpHH33U/uY3vym3D+82q1atKv3ZLrzwQnv69GkbHR1drt6lS5fa8ePH21deecVOnTrVFhUVWWutLSoqslFRUWf8fNaW/zfnBaTYKnK2Xhf9sNbOB+aDM0xdn/sWabKOHYAXJsOJXLjgf6HHKMhYDl8+BVmb4JaPILyz21U2aUuWLGHNmjWMGDECgFOnTtG5c2eSk5Pp1asX55133hmvGTp0KAcPHiQzM5OcnBw6dOhAjx49AOdc7VtvvQXA3r172b59O126dKly/1W19fTTT1daV1nLly/nqquuok2bNgBMnz6dZcuWMXTo0Cr3FxoayqlTp7j33nu5+eabSUpKIj8/n4KCAtq1a8ebb75JQkJCaRuDBg2ic+fO5Ofnc+TIEe6++27AuYTo1ltvpWvXrqXthoWF8eabbzJ27FgSExNLX79w4cJy+3jjjTdISkpi5MiRAMTFxdGyZUtyc3Np3759uXqTk5Ox1jJnzhyWLFlCaGhouf3l5eURERFR5c/rC3+E8X6gR5nvYzyPiYjbCvPhjR/AiUMw8z3oPtx5/Jzx0PdCePkq5/mZH0Bo01iQ73eXx7ldQqldu3YRGhpKx44dufnmm3nkkUfKPZ+RkVEacpWZMWMGCxYsICsri2uuuQZwgv3TTz9lxYoVtG7dmgkTJvi0AEVlbVlrK62rrlq3bs2mTZt49913ue2225g1axajRo1i0KBBAGzcuLE0SAE2bdrEJZdcwubNmxk2bFjpefMNGzZw++23A84qa926dcMYQ1paGoMHDy59fWpqKoMGDWLz5s2l+0hNTWX48OGl26xZs4YJEybQqlWrM45XamoqBw4cIDIy8ozQPX36NC1btqzzMfHHpU0LgZs8s6rPA45aaw/4oV0RqasvHoP9KTD9798FsVev8+GKv8DeVbDymcpfLwGTk5PD7NmzueOOO7joootYsGABBw8eBODw4cPs3l3lrW9LXXPNNbz++ussWLCAGTNmAHD06FE6dOhA69at+eabb1i5ciUAERER5OXl1agtX+oaN24cb7/9NidPnuTEiRO89dZbjBs37qx1b9++nTZt2nDttddy2WWXkZ+fT2pqKkOGDAEgMjKSbdu2AbB+/XpeeeUVEhISSE1NJSEhobSdjRs3lr5mw4YNpV93796dtLQ0wHnD8/LLL3PTTTedsQ/vZLY1a9bw73//m4SEBDp06EBxcXFpIB84cIAbbriBd955h/DwcD766KPS/R86dIioqKgarUFdFV8ubfo3sAIYYIzZZ4y51Rgz2xgz27PJB8AuYAfwD+Anda5KROoudzt8OQ+GXAuxl1e+zeCrYeBl8PnD8G31f/ylbk6dOlV6adPEiROZPHkyv/vd7xg0aBAPPfQQkydPZsiQIUyaNIkDB6rv08TFxZGXl0f37t1Lh2ovueQSioqKiI2N5b777isd4o6MjGTMmDHEx8eXTuCqri1f6ho2bBgzZ85k5MiRjBo1ilmzZp11iBrg4YcfZsCAAQwbNoz09HR+8pOflAvKG2+8kZSUFAYPHszzzz9P7969Oeecc0hNTS3tMefn53Pq1Ck6dOgAlA/mG2+8kczMTAYPHsy1117LCy+8QGRk5Bn7WL9+PYmJiTz22GO0b9++tNc8efJkli9fzsmTJ5k+fTpPPPEEsbGxPPDAA+Vmvi9evJhLL7202t+TL6q9tClQdGmTSIC98QPYtRTuXHP2c8JH98O8oTB4BlzZOHvIDeXSJvHN8ePHS2d6P/744xw9epSHHnqo3va/du1a5s6dy8svv3zW7aZPn86jjz5K//79z3jO75c2iUgQyk6DLe/CebdXPzmrXXcYMQs2vAa5O+qnPpGzmDt3LnFxcSQmJpKRkcEDDzxQr/sfNmwYF1xwQbWLflx55ZWVBnFtqGcs0hj9ZxZs/RDuToXWHavf/ngOzI2DoT+Ay+YEvr56pp6x1Df1jEWauuMHYfNbMOwm34IYILyTM0y94d9w6khg6xORMyiMRRqbdS9DSREk3VKz1436MRSehPV+WlFIRHymMBZpTEpKYM2/oE8yRJ1bs9d2HQLdk2D9vwNSmohUTWEs0pjsXemsPz30ptq9PuFayE6F7M3+rUtEzkphLNKYbPoPNGsFA6bU7vVxV0FIM9j4hn/rEpGzUhiLNBbFRbD5bRhwCbQIr10bbaKg30TY+H9QUvVlHSLiXwpjkcYifSmczIX4q+vWzpDvQ16ms0ymiNQLhbFIY7H5LWjR1unZ1sW5kyE0DL553z91iUi1FMYijUFJCWz7GM6dBM3reAeZFhHObOytH4BLiwI1VqGhoaVrUyckJPDEE09QUlJSozZGjx5dq30fOXKEZ5991i9t+WrevHnExsZyww03lHv8ueeeIzExkcTEREJCQkq/vueee2q1n88++4wbb7zxjMdPnTrF+PHjq11JKzk5maKiolrt218UxiKNQeY6OHEQ+l/in/YGTIHDuyBnq3/aEwBatWrF+vXr2bx5M4sWLeLDDz8sd+OBs7HWUlJSwldffVWrfVcWxrVty1fPPvssixYt4tVXy1+7PmvWLNavX8/7779Pjx49WL9+PevXr2fu3Lm12s+GDRsqvTnFCy+8wPTp00vvP1yZsLAwLrroIt54w91JiwpjkcZg20dgQuo+RO01YKrzeauGqgOlc+fOzJ8/n6effhprLa+88gojR44kMTGRH//4xxQXF5ORkcGAAQO46aabiI+PZ+/evaU3ULjvvvt45pnvbuzx4IMP8uc//xmAK6+8kuHDhxMXF8f8+fNLt9+5cyeJiYmld23ypa3K6qpozpw5xMfHEx8fz5NPPgnA7Nmz2bVrF1OmTKkyZDdt2lTuvsMACxYs4LzzziMhIYGxY8eSk5MDODdluP/++0lOTqZnz558+umnpa/ZsGEDWVlZZzz36quvMm3atNLtLrjgAhYtWgTA/fffz5133ll6vCq+Yah31lpXPoYPH25FxE/+Otba5y/2b5t/H2/tPy7yb5suSUtL++6bD35l7QtT/fvxwa98qqNNmzZnPNauXTu7ZMkSe9lll9mCggJrrbW33367ffHFF216ero1xtgVK1ac0cbatWttcnJy6eOxsbF2z5491lprDx06ZK219uTJkzYuLs7m5uba9PR0GxcXV2k9VbWVlpZWaV1lpaSk2Pj4eHv8+HGbl5dnBw0aZNeuXWuttbZXr142JyenyuPx+OOP21/9qvyxy83NLf36wQcftE8//bS11tp+/frZxx9/3Fpr7X//+187c+bM0u0SEhLsY489Vu6506dP2+jo6HJtL1261I4fP96+8sordurUqbaoqMhaa21RUZGNioqqss7aKPdvzgNIsVVkYjN33wqISJ0dy4SsjTDxQf+2228SLPsznPoWWnXwb9tSzpIlS1izZg0jRowAnHOdnTt3Jjk5mV69epXek7isoUOHcvDgQTIzM8nJyaFDhw706NEDcM7VvvXWWwDs3buX7du306VLlyr3X1VbTz/9dKV1lbV8+XKuuuoq2rRpAzg92GXLllV7T2NwesaTJk0q99i//vUv3njjDU6fPk1WVhZ/+tOfOHnyJEePHi09p1xYWEj79u1Lvz506BD33ntvuedyc3NLt/FKTk7GWsucOXNYsmRJ6fB1aGgoYWFh5OXlERERUW3dgaAwFgl2Oz93PvebdPbtaqrvhfDFY5C+DAZd4d+23TTlUbcrKLVr1y5CQ0Pp2LEjN998M4888ki55zMyMkpDrjIzZsxgwYIFZGVlcc011wBOsH/66aesWLGC1q1bM2HCBPLz86utpbK2rLWV1uUvqamp5SZtvfTSS3z99dd8/vnnhIeHk5ycTFxcHGlpaQwfPrw0PDdu3Eh8fDzg3B0pISGBkJCQcs+1atXqjJ87NTWVAwcOEBkZeUbonj59mpYt6zj5sQ50zlgk2KV/Aa2jIDrOv+3GJEFYxHdhL36Vk5PD7NmzueOOO7joootYsGABBw8eBODw4cPs3r272jauueYaXn/9dRYsWMCMGTMAOHr0KB06dKB169Z88803rFy5EoCIiAjy8vJq1JYvdY0bN463336bkydPcuLECd566y3GjRtXbe0lJSVs37693G0GU1NTGT16NOHh4fznP//hq6++YvDgwaSmppKYmFi63caNGxkyZAjgnC9OSEg447kOHTpQXFxcGsgHDhzghhtu4J133iE8PJyPPvqo9DWHDh0iKiqK5s2bV1t3oCiMRYKZtbBrqXMpkjH+bTu0OfQeC7sW+7fdJuzUqVOllzZNnDiRyZMn87vf/Y5Bgwbx0EMPMXnyZIYMGcKkSZM4cOBAte3FxcWRl5dH9+7d6dq1KwCXXHIJRUVFxMbGct9995UOcUdGRjJmzBji4+NLJ3BV15YvdQ0bNoyZM2cycuRIRo0axaxZs3waot6xYwcxMTGEhYWVPjZz5kyeffZZRo4cybp16zjnnHNo06bNGWG8adOm0p7xhg0bSoO54nOTJ09m+fLlnDx5kunTp/PEE08QGxvLAw88UG4W++LFi7n00kurrTmQjHXpOsKkpCSbkpLiyr5FGo2crfDMSLj8KRg+0//tr5oPH/4CfrYOOp7j//brSWU3epfGb+3atcydO5eXX375rNtNnz6dRx99lP79+/tt35X9mzPGrLHWJlW2vXrGIsFs11Lnc5/xgWm/7wWe/SwJTPsiATRs2DAuuOCCahf9uPLKK/0axLWhMBYJZulLoX1P6NgnMO1H9oPwLrA7sItDiATKLbfcUu2iHzfdVMtbjvqRwlgkWJUUQ8Yy53xxoBgDvUZDxpdaGlMkgBTGIsEqayPkH4U+EwK7n16jnbs4Hal+dq+I1I7CWCRY7V7hfO49JrD76eVpP8iHqt2arCpNT23+rSmMRYLV3pXQrie07RbY/XQa6KzAtfvLwO4ngFq2bMmhQ4cUyBJw1loOHTpU4wVEtAKXSDCyFvasgj7VL65QZyEh0HN0UPeMY2Ji2LdvX+lNB0QCqWXLlsTExNToNQpjkWB0ZDccz4Ieo+pnf71GO3dwOnYA2natn336UfPmzenTJ0AzzkX8QMPUIsFozyrnc88zbyAQEL08N6EP4qFqkYZMYSwSjPauctaN7jyofvbXZQg0bw37VtfP/kSaGIWxSDDau8q5kUNI1YsZ+FVoM+g2VGEsEiAKY5Fgk38UsjfX3xC1V0wSZKVC0en63a9IE6AwFgk2+1YDtv4mb3nFjIDiAjiwsX73K9IEKIxFgs2eVWBCnJ5qferu2Z+GqkX8TmEsEmz2fQ3RcdAion7327YrtI2B/br1qYi/KYxFgom1kLkOug93Z/8xSeoZiwSAwlgkmBze5Uzg6jbMnf3HjIAjeyAv2539izRSCmORYLJ/rfO5u1th7DlvrKFqEb9SGIsEk8y10KwVdIp1Z/9dEyCkGexTGIv4k8JYJJjsXwtdhziLcLiheSvoMljnjUX8TGEsEiyKi+DABvfOF3t1T3ImkZWUuFuHSCOiMBYJFjnfQNEp984Xe3VLhILjcHinu3WINCIKY5FgkbnO+dxtqLt1dE1wPmeud7cOkUZEYSwSLDLXQou20LGvu3V0GgihLeCAwljEXxTGIsFi/1pniDjE5f+2oc2hS7x6xiJ+pDAWCQZFp507Nbk9ecura6IzmUyTuET8QmEsEgyyNkFJofuTt7y6JUJBnrMimIjUmcJYJBhkelbeakg9Y9B5YxE/URiLBIMD66F1JLSLcbsSR+dYZxKXd4a3iNSJwlgkGGRtcla+MsbtShyhzZ3bOB7Y4HYlIo2CwlikoSsuchb8iI53u5LyuiXCgY3ObR1FpE58CmNjzCXGmK3GmB3GmPsqeb6nMWaxMWadMWajMWaq/0sVaaIO74Si/IYXxl0T4fRRTeIS8YNqw9gYEwo8A0wBBgHXGWMGVdjsfuBNa+1Q4FrgWX8XKtJkZaU6n7s0sDDupklcIv7iS894JLDDWrvLWlsAvA5Mq7CNBdp6vm4HZPqvRJEmLmSEJ30AACAASURBVHuzc9vCqP5uV1Jep1gIDdPiHyJ+4Mt92LoDe8t8vw8YVWGbB4FPjDF3Am2AiX6pTkQgexNEDYBmLdyupLxmYZ5JXApjkbry1wSu64B/WWtjgKnAy8aYM9o2xtxmjEkxxqTk5OT4adcijVzWpoY3RO3VZbBTnyZxidSJL2G8H+hR5vsYz2Nl3Qq8CWCtXQG0BKIqNmStnW+tTbLWJnXq1Kl2FYs0JScPQ16m0wNtiKIHw6nDkHfA7UpEgpovYbwaONcY08cYE4YzQWthhW32ABcBGGNiccJYXV+Rusre5HxuaDOpvboMdj57J5mJSK1UG8bW2iLgDuBjYAvOrOnNxpg/GGOu8Gx2L/AjY8wG4N/ATGs1biVSZ1meMPaGXkPj7bErjEXqxJcJXFhrPwA+qPDYb8t8nQaM8W9pIkL2ZmjTCcI7u11J5Vq2hQ69FcYidaQVuEQasuzUhjtE7RUd/91wuojUisJYpKEqLoKD3zTcmdReXYbAoZ1QcMLtSkSClsJYpKE6tAOKTzf8nnGXeMBCdprblYgELYWxSEPV0GdSe5XOqN7obh0iQUxhLNJQZaVCSPOGtwxmRe16QMt2Om8sUgcKY5GGKnszdBrgLDvZkBnj9N41o1qk1hTGIg1V9qaGP0Tt1WWwc864pNjtSkSCksJYpCE6cchZYrKhLoNZUXQ8FJ6Aw+luVyISlBTGIg2R9/xrQ7+sycs7iStbQ9UitaEwFmmISmdSN9BlMCvqNBBMqM4bi9SSwlikIcreDG06Q3iQ3N2seUtnslmWZlSL1IbCWKQhykoNniFqL82oFqk1hbFIQ1NcCDnfBM9Maq8ug517L5845HYlIkFHYSzS0BzaAcUFQRjGnno1iUukxhTGIg1NVpDNpPbyTjbTeWORGlMYizQ02UGyDGZF4Z0gPNqZfCYiNaIwFmlosjc7lwqFNne7kpqLjtcwtUgtKIxFGpqsTcE3RO0VHQc5W51JaCLiM4WxSENyIheOZwXf5C2vLoOdyWe5292uRCSoKIxFGpLSlbeCZE3qirx167yxSI0ojEUaktKZ1EGyDGZFUf2dyWc6byxSIwpjkYYkexOEd4E2UW5XUjuhzZ3JZ7q8SaRGFMYiDUn2puAdovbqEq9hapEaUhiLNBTFhc5M5GCdSe0VHe9MQjuR63YlIkFDYSzSUORuC85lMCsqncSloWoRXymMRRoK79BusIdxFy2LKVJTCmORhiIrFULDIOpctyupmzZRWhZTpIYUxiINRfYm6DQgOJfBrEjLYorUiMJYpKHI3vzdnY+CnZbFFKkRhbFIQ3A8B45nB/9Mai8tiylSIwpjkYbAO6Qb7NcYe3knoWlGtYhPFMYiDUHpTOpGMkwdda4zGU1hLOIThbFIQ5C1CSK6QptItyvxj9DmzmQ0Xd4k4hOFsUhD0BiWwawoWstiivhKYSzitqICZ+ZxsC/2UZGWxRTxmcJYxG2526CkMHhvm1gVLYsp4jOFsYjbvGHV2IaptSymiM8UxiJuy94EoS0gMsiXwayodFlMhbFIdRTGIm7L2gSdB0JoM7cr8b/oeIWxiA8UxiJuy97U+CZveXWJ17KYIj5QGIu46fhBOJHTeMM4Ol7LYor4QGEs4qYszzKYjWVN6oq0LKaITxTGIm4qXQazkYaxlsUU8YnCWMRN2Zsgohu07uh2JYGhZTFFfKIwFnFTViNcBrMiLYspUi2FsYhbigogd2vjPV/spWUxRaqlMBZxS+5WKClqvOeLvbxvNryT1UTkDApjEbd4z6M29jAunVGtoWqRqiiMRdxSugxmP7crCaw2URDeRTOqRc5CYSziluxN0Dm2cS6DWVF0nMJY5CwUxiJusNYzk7qRD1F7aVlMkbPyKYyNMZcYY7YaY3YYY+6rYpvvG2PSjDGbjTGv+bdMkUbm+EE4mdv4Z1J7aVlMkbOqdnzMGBMKPANMAvYBq40xC621aWW2ORf4NTDGWvutMaZzoAoWaRSyPTOLm0rPuOyymNGD3K1FpAHypWc8Ethhrd1lrS0AXgemVdjmR8Az1tpvAay1B/1bpkgjUzqTupEv+OHlXRZTlzeJVMqXMO4O7C3z/T7PY2X1B/obY740xqw0xlxSWUPGmNuMMSnGmJScnJzaVSzSGGRvhrbdG+8ymBV5l8XU5U0ilfLXBK5mwLnABOA64B/GmPYVN7LWzrfWJllrkzp16uSnXYsEocZ8D+OqRA/WjGqRKvgSxvuBHmW+j/E8VtY+YKG1ttBamw5swwlnEamo6DTkbms6Q9Re0XFwPBuOa1RMpCJfwng1cK4xpo8xJgy4FlhYYZu3cXrFGGOicIatd/mxTpHGI+cbZxnMpjKT2quL7m0sUpVqw9haWwTcAXwMbAHetNZuNsb8wRhzhWezj4FDxpg0YDHwC2vtoUAVLRLUvJO3ugxxt476pmUxRark09I/1toPgA8qPPbbMl9b4OeeDxE5m6xUaNYKOp7jdiX1S8tiilRJK3CJ1Ldszz2MQ0LdrqT+dYlXGItUQmEsUp+shayNTe98sVd0nJbFFKmEwlikPh3dB/lHoctgtytxR/Rgz7KY29yuRKRBURiL1CfvClTRTTWMPZdzaRKXSDkKY5H6lL0JME13fWYtiylSKYWxSH3K2ggd+0CLCLcrcYeWxRSplMJYpD5lbWq654u9tCymyBkUxiL1Jf8YfJvedM8Xe3WJ17KYIhUojEXqy0HPLcCbfM/YO4lLvWMRL4WxSH3xTlpqqtcYe2lZTJEzKIxF6ktWKrTq4NzHuCnTspgiZ1AYi9SXrFSnV2iM25W4r0v8dzfMEBGFsUi9KC5yzhk3tTs1VSU6zrmVpJbFFAEUxiL14/BOKMrX+WKv6MFQUqhlMUU8FMYi9aF08lYTn0nt1UWTuETKUhiL1IesVAhpDlED3K6kYYjsp2UxRcpQGIvUh6xU6DQQmoW5XUnDENrcOR7qGYsACmOR+pG9SeeLK4qO1+VNIh4KY5FAO37QWf5R54vL07KYIqUUxiKBlrXR+RytnnE5WhZTpJTCWCTQDmxwPndNcLeOhsZ7wwyFsYjCWCTgMtdDh97Qqr3blTQsbSIhoqsmcYmgMBYJvAMboGui21U0TNFxWhZTBIWxSGCd+haO7NYQdVWi451lMYtOu12JiKsUxiKBpPPFZ9c1wVkW03uvZ5EmSmEsEkilYaxh6kp1G+p8zlzvbh0iLlMYiwTSgQ3QroczWUnO1KE3tGwPBxTG0rQpjEUC6cAGDVGfjTHO8clc53YlIq5SGIsESv4xOLRDQ9TV6TYUstM0iUuaNIWxSKB470iknvHZdUvUJC5p8hTGIoGimdS+8Y4caKhamjCFsUigHNjgrDAVEe12JQ2bdxKXZlRLE6YwFgmUA+vVK/aFMc5QtWZUSxOmMBYJhIITkLtNYeyrromaxCVNmsJYJBCyN4Mt0UxqX3Ub6kzi0k0jpIlSGIsEgvf8Z9ch7tYRLLp53rRoqFqaKIWxSCBkroU2naFtd7crCQ7te2kSlzRpCmORQNi/BroPdyYnSfW8k7h0eZM0UQpjEX/LPwq5250wFt91GwoHt2gSlzRJCmMRf8tcD1joPsztSoJL10RN4pImS2Es4m/71zifvbcHFN9000pc0nQpjEX8bf8a6HgOtO7odiXBpX0vaNVBYSxNksJYxN/2r9X54towxjlu+9e6XYlIvVMYi/jTsQOQl6kwrq3uSc7dm07nuV2JSL1SGIv4U6anV6cwrp2YJMBqqFqaHIWxiD/tXwMhzaDLYLcrCU7eNzH7UtytQ6SeKYxF/Gn/Gug8CJq3cruS4NS6I3TsqzCWJkdhLOIvJSWwf52GqOsqJgn2p4C1blciUm8UxiL+cngnnD6qMK6rmBFwPBuO7nO7EpF6ozAW8RfvYh9aeatuSs8br3a3DpF65FMYG2MuMcZsNcbsMMbcd5btvmeMscaYJP+VKBIk9q+F5m2g00C3Kwlu0fEQ2uK7NzciTUC1YWyMCQWeAaYAg4DrjDGDKtkuArgLWOXvIkWCwr7VzpKOIaFuVxLcmoU5x1GTuKQJ8aVnPBLYYa3dZa0tAF4HplWy3R+B/wfk+7E+keBQcBKyNkKPkW5X0jh0T4ID66G40O1KROqFL2HcHdhb5vt9nsdKGWOGAT2ste+frSFjzG3GmBRjTEpOTk6NixVpsDLXQUkR9BjldiWNQ8xwKMqH7E1uVyJSL+o8gcsYEwLMAe6tbltr7XxrbZK1NqlTp0513bVIw7HXc3YmRj1jv4gZ4XzWULU0Eb6E8X6gR5nvYzyPeUUA8cASY0wGcB6wUJO4pEnZ+zVE9oM2kW5X0ji06wFtOiuMpcnwJYxXA+caY/oYY8KAa4GF3iettUettVHW2t7W2t7ASuAKa63+F0nTYK3TM+5xntuVNB7GOIt/6PImaSKqDWNrbRFwB/AxsAV401q72RjzB2PMFYEuUKTBO7QTTh3W5C1/6zHKWUjlRK7blYgEXDNfNrLWfgB8UOGx31ax7YS6lyUSRLznizV5y796nu983rMSYi9ztxaRANMKXCJ1tXcVtGwHUf3drqRx6ZboLP6xZ4XblYgEnMJYpK72fu3Mog7Rfye/atbCWRpzz0q3KxEJOP31EKmLU0cgZ4uGqAOl53nO4h8FJ92uRCSgFMYideG99EaTtwKj5/nOYipap1oaOYWxSF3sXQkmRLdNDJQeIwCjoWpp9BTGInWx+yvoMgRahLtdSePUqgN0HqRJXNLoKYxFaqsw3xmm7j3W7Uoat57nOZPkSordrkQkYBTGIrW1fw0Un4ZeY9yupHHreT4U5EH2ZrcrEQkYhbFIbe3+EjBOz00Cp6dnprrOG0sjpjAWqa2M5RAdB607ul1J49auB7TtDnu+crsSkYBRGIvURlGBcx5TQ9SBZwz0Gg0ZXzo35RBphBTGIrVxYD0UnYLeCuN60XscnDgIudvcrkQkIBTGIrWRsdz5rJ5x/eiT7HxO/8LdOkQCRGEsUhu7v4SoAdAmyu1KmoYOvZ1zxwpjaaQUxiI1VVwEe1ZpiLo+GeMMVWcsh5ISt6sR8TuFsUhNZW10rnvVEHX96jMOTh2Gg2luVyLidwpjkZrKWOZ81spb9av3OOez9/iLNCIKY5Ga2rUEOg2EiC5uV9K0tO/hnDtOVxhL46MwFqmJwnzn5hDnXOB2JU1T73Gwe7nWqZZGR2EsUhN7V0FRPpwzwe1KmqY+yZB/FLJS3a5ExK8UxiI1sWsxhDTTTGq3eM8b6xInaWQUxiI1sWsJxIyAFhFuV9I0te0KUf2dN0UijYjCWMRXJw9D5nqdL3Zb34uc8/aFp9yuRMRvFMYivkr/ArA6X+y2fhOd8/a7v3S7EhG/URiL+GrXYgiLgO7D3a6kaes1GkJbwI7P3a5ExG8UxiK+2rXEWQUqtJnblTRtYa2dQN7xqduViPiNwljEF4fT4dsMnS9uKPpdBLlb4eg+tysR8QuFsYgvvL2wvgrjBqHfROfzjs/crUPETxTGIr7Y/gl06AOR/dyuRMCzHGk32KkwlsZBYSxSnYKTzkzq/hc7t/IT9xkD/S6EnUucW1qKBDmFsUh1MpY5l9KcO9ntSqSsvhfB6aOwf43blYjUmcJYpDrbP4HmrXXLxIbmnAlgQmH7x25XIlJnCmORs7EWtn3i/OFv1sLtaqSs1h2h5/mw9SO3KxGpM4WxyNnkbIWjezRE3VANmAIHNzuXnYkEMYWxyNl4h0AVxg3TgCnOZ/WOJcgpjEXOZutHEB0P7bq7XYlUJrIvRA2ArR+4XYlInSiMRapy/CDsWQEDL3O7EjmbAVOcm0acOuJ2JSK1pjAWqcrWDwALsZe7XYmczYCpUFKktaolqGnFe5GqbHkPOvSG6Di3K/FZSYll77cn2ZqVx86cE2Qfyycn7zQ5eac5VVhMQVEJBcUlhIYY2oSF0ioslPatwujWvhXd2rekR8fWxHZpS4+OrTDBssBJTBK0joKtH8Lgq92uRqRWFMYilck/6tyl6bzZDXrVreOni0jJOMzqjMN8nX6YTfuPcaqwuPT58BbN6Ny2BZ3CW9ApogVhoSE0bxZCcUkJJwuKOXm6mB05x1m6Lafc6yJaNCO2a1tG9OnA6L5RDO/VgZbNQ934EasXEgoDLoG0hVB0WpegSVBSGItUZvsiKCmEgQ1viDr7WD6L0rL5JC2bFTtzKSy2NAsxxHdvx3UjezKwSwTnRofTr3M4ES2b+9SmtZajpwrJOHSSLQeOkZZ5jNT9R/nb0l08s3gnYaEhjDqnI1PiuzJpUDSdIhpY4A26Eta9AjsXO8EsEmQUxiKV2bIQwqMhZoTblQBwsqCID1KzeDNlL1+nHwagd2RrfjimD8nndmJYr/a0Dqv9f2djDO1bh5HYOozEHu1LHz9+uojVGYf5cnsun27J5jdvpXL/26mM6hPJ90fEMCW+a8PoMfcZDy3bQdrbCmMJSsZa68qOk5KSbEpKiiv7FjmrwlPwWF9IuAYum+tqKWmZx3jxqwze25jJiYJieke25nvDYrgkvgv9OofX63lday3fZOXx4aYs3lm/n92HThLRshlXJnbnB+f1YkCXiHqrpVJv3Q7fvA+/2K6hammQjDFrrLVJlT2nnrFIRds/gcITEHuFK7svKbEs3Z7D88vSWb4jl1bNQ7k8oSszknqQ1KuDaxOrjDHEdm1LbNe23H3RuaxMP8Sbq/fyZspeXl65m/H9O/Gjcecwpl+kOzXGXQkbXnPO9fe/uP73L1IHCmORilIXQJvO0Ce5XndbXGJ5b2MmT3++g+0HjxPdtgW/umQg14/sSbvWvp37rS8hIYbRfaMY3TeK350o4LWv9/DPLzP4wfOriO3alrsu6sfkQV0ICanHUD7nAmjRDja/rTCWoKMwFikr/5jTMx52kzNLtx6UlFg+2pzFk59uY1v2cfpHhzPn+wlcNqQbYc0a/lIAHdqE8dML+jFrXB/eWZ/J35buZPYraxnUtS0/n9Sfi2I7109PuVkYDJwKW9+HogLne5EgoTAWKWvrB869i+Pr53rVL7bl8MiH37DlwDH6dmrDX64byqWDu9Zvj9JPWjQL5ftJPZg+tDsLN2Ty1GfbmfVSCgkx7bhvSizn940MfBGDroQN/4adn2silwQVhbFIWakLoF1P6DEyoLvZmXOch9/fwuffHKRnx9bMvSaBKxK6ExqEIVxRs9AQpg+L4fKEbvx37T6e+nQ71/1jJRfHRfObqbH0imwTuJ33vRBadYSNbyiMJagojEW8ThyCXYvh/DsCttDHkZMFPPXZdl5esZtWzUP5zdSB3Dy6Ny2aNYDLg/yseWgI14zoybTE7jy3bBfPLtnJxDlLmTm6N3dceC7tWgXgPHizMIif7lxznH/UudxJJAg0/BNSIvUl7W1njeP47/m96ZISyxur9zDhz0t48asMvj+iB4t/MYHbkvs2yiAuq2XzUO648FyW/M8ErhraneeWp3PRE0t5Z/1+AnJp5ZBrnVMNaQv937ZIgPgUxsaYS4wxW40xO4wx91Xy/M+NMWnGmI3GmM+MMb38X6pIgG14HToNhC6D/drs9uw8rp2/kl/9J5X+nSN4/2fj+NNVg4kKb1rXwnZu25LHrk7g3TvG0r19S+56fT03vfA16bkn/LujmCTo2NcZqhYJEtWGsTEmFHgGmAIMAq4zxgyqsNk6IMlaOwRYADzm70JFAipnK+z7GhJv8NsQdX5hMX/+eCtT5y1j28E8HvveEF6/7Txiu7b1S/vBKr57O/77kzH8cVoc6/cc4eInv+CpT7dzuqi4+hf7whgYcg1kLIMje/3TpkiA+dIzHgnssNbustYWAK8D08puYK1dbK096fl2JRDj3zJFAmz9q2BCIeFavzS3YuchLn7yC55evIPLE7rx2c/H8/0RPYJylnQghIYYbjy/N5/dO57Jg6KZ++k2pjy1jDW7D/tnB0O+73xOfdM/7YkEmC9h3B0o+/Zyn+exqtwKfFiXokTqVXGRM0Td/2II71ynpvILi/nje2lc94+VGOC1WaOY8/1EIpvYkLSvOrdtydPXD+PFW0ZyurCEq/+2gofeS+NUQR17yR37QM/zYf1r4NKSvyI14dcJXMaYHwBJwONVPH+bMSbFGJOSk5Pjz12L1N6OT+F4Ngz9QZ2aSd13lMv/spznl6dz0/m9+OCucYzuF+WnIhu38f078fE9yVw/sifPLU9n6rxlpGTUsZc87GY4tAMylvunSJEA8iWM9wM9ynwf43msHGPMROB/gSustacra8haO99am2StTerUqVNt6hXxv3UvQ5tOcO7kWr28qLiEeZ9t56pnvyQvv4iXbhnJH6bF1+kuSk1ReItmPHzVYF6dNYqCohJm/H0Ff6xLLznuSufSpjX/8mudIoHgSxivBs41xvQxxoQB1wLlrhkwxgwF/o4TxAf9X6ZIgORlwbaPnAk/oTW/7nVnznG+97cVzFm0jUuHdOXju5NJ7q83mnUxpl8UH9+TzA9G9eL55elMeeqL2p1Lbt7Kucxpy0LnGnKRBqzaMLbWFgF3AB8DW4A3rbWbjTF/MMZ4b2vzOBAO/J8xZr0xRhf4SXBY86JzbXHSLTV6WUmJ5cWvMrh03jJ2HzrB09cP5alrhza4GzoEq/AWzfjjlfG89qNRFJVYZvxtBY9//A0FRSU1a2j4zVBc4CyRKdKA6X7G0nQVF8KTg6FzLNz4ls8vO3D0FL/4v40s35HLhAGdeOx7Q+jctmUAC23a8vIL+eN7abyZso/47m158ppE+nWuwb2Tn5sEp76FO1YHbGU1EV+c7X7GWoFLmq5v3oe8AzDyNp82t9by9rr9TJ77BWv3fMufrhrMP2eOUBAHWETL5jx2dQJ/+8FwMo/kc+m85fzzy3RKSnzsSAyfCYe2Q/rSgNYpUhcKY2m6Vj/n3BTCh4lbh08U8NPX1nL3G+sZEB3Bh3eN4/pRPevn1oACwCXxXfjo7nGM6RfF799N4+Z/fk3W0fzqXxj/PWgdBSv/GvgiRWpJYSxN08EtzgpNI26p9r7Fn3+TzcVPfsGitGx+dclA3vjx+YG985BUqXNES56/OYmHr4onJeNbLn7yC97dkHn2FzVvCSNudSbqHdpZP4WK1JDCWJqmlX+FZi1h6E1VbnLidBG//m8qt/wrhcg2Ybzz07HcPqFvo7jNYTAzxnDDKOc67j5Rbbjz3+u46/V1HD1VWPWLkm6F0DD1jqXBUhhL05OX5cyuTbwB2lR+w/vVGYeZ8tQyXl+9h9nj+/LOHWMY1K1prynd0PSJasOC2edzz8T+vLfxAJc8+QVf7citfOOIaIi/2ln29NS39VuoiA8UxtL0rPqbcznT+T8946nTRcU8+uE3fP/vKwB488fnc9+UgY3+NofBqlloCHdNPJf/3j6aVs1Duf65Vfzh3TTyCytZKOS826HwpHM5m0gDozCWpiX/GKx+AWKvgMi+5Z5KyzzGtKe/5G9Ld3LtiJ58cNc4RvTu6FKhUhMJPdrz/s/GcdP5vXjhy3Qu+8tyNu0/Wn6jrkOgz3hY+SwUnnKnUJEqKIylaVn7Ipw+CmPuKn2ouMTy7JIdTHtmObnHC3hhZhKPTB9MeAstZxlMWoWF8odp8bx0y0jy8gu58pkv+ctn2ykqLrNQSPIvnHXI173iXqEilVAYS9NReApWPAO9x0H3YQDsPnSCa/6+gsc+2sqkQdF8ck8yFw6MdrlQqYvk/p34+O5kpgzuyhOLtjHj7ytIzz3hPNl7LPQ4D5Y/CUUF7hYqUobCWJqOlH86i3yM/xXWWl5euZtLnlzG1uw8nrwmkWeuH0bHNmFuVyl+0L51GH+5bijzrhvKzoPHmfrUMl5euRsLTu/42D7Y+LrbZYqUUhhL01BwApbPgT7JZHZI4qYXvuaBtzcxok9HPrknmSuHdtcCHo3QFQnd+OSe8ST17sADb29i5j9Xk915DHRNhGVznCVRRRoAhbE0DV//A07ksLjbj7h47hes2e0sZ/niD0fQtV0rt6uTAOrSriUv3TKSP06LY1X6IS5+ahmre98G36Y7t88UaQAUxtL4nc6jZPlTbGo1gh9+Fkpst7Z8dFeylrNsQowx3Hh+b97/2Th6RbZhxuJ27GoVT8niR51RExGXKYyl0dv+1p8IyT/Mg3nTeOCyQbz+o/PoGdna7bLEBX07hfOf2efz80kDuO/Y1YScyGbbwsfdLktEt1CUxisn7zRP/mcx96ffyOqW59Pt1tfo1znc7bKkgdi0/yh5/5pBXMFGHu73b3551Wgiw1u4XZY0YrqFojQp1lreTNnLxDlLGbnrLzQPNYy+bZ6CWMqJ796OpFvmEm5OE7ftb0ycs5R31u/HrQ6KNG0KY2lU9hw6yY3Pf80vF2xkaodMpoUsp9mYO2gW2dvt0qQBat41jpARt3Bjs09IbneQu15fz6wXU3y7NaOIHymMpVEoLrE8t2wXFz/5Bev3HuGhaYP4U6tXIDwaxt7jdnnSkF3wv5iW7Xky/BXunzqQL3fmMmnOUl5btYeSEvWSpX4ojCXobc48yvRnv+Sh97cwum8ki36ezA9CP8XsXw2T/ggtItwuURqy1h1h4oOYvSuY1S6Fj+5KZlC3tvzmrVSu/ttXpGUec7tCaQI0gUuC1rH8QuZ8so2XVmTQoXUYv7sijsuHdMXkZcEzI50lL298G3T5klSnpASeuwiO7oOfrsK26sB/1u7nTx9s4eipQmaO7s09k/prvXKpE03gkkbFWstb6/Zx4Z+X8uKKDG4Y1YvP753AFQndnOuGP/wlFBfApXMUxOKbkBC4/Ck4dRg++jXGGK4eHsPn947n+0k9eH55OhOfWMqHqQc0wUsCQmEsQWVrVh7XzF/JPW9soHuHViz86Vj+eGU87Vo3dzbY/DZsWeisP1zhFokiZ9V1CIy711mzeutHgLPGN2cQxAAAFedJREFU9SPTB/Of20fToU0Yt7+6lh/+azW7co67XKw0NhqmlqBw5GQBT322nZdW7CaiZTN+dclArknqQUhImZ7vsQPw1/OhQ2+4dRGENnetXglSRQUwf4LTQ/7JCmjV4bunikv411cZzF20jYLiEm4+vzd3XnQu7Vrp35n45mzD1ApjadAKikp4eeVu5n22nbz8Qq4Z0ZNfXDzgzLsrWQuvfA92fwWzl0HUue4ULMEvcx08NxEGXgYz/nXGqY6DefnM+WQbb6TspUPrMO6Z1J/rRvSgWagGGuXsdM5Ygo61lo83ZzF57lL++F4aQ2La8cFd43hk+uDKb3P49XzY+RlM/qOCWOqm21C48H5IexvW/POMpztHtOTR7w3hvTvHcm7ncB54exOXzlvO8u25LhQrjYV6xtLgbNx3hIff38Kq9MP06xzO/14ay4T+naq+qcPe1fDPKdD3Qrj+DU3akrorKYFXr4bdX8KPPofouEo3c940ZvOnD7aw5/BJkvt34pcXDyC+e7t6LliCgYapJShszcpjzqKtfLw5m45tfBz+O54D88dDSDP48dJy5/hE6uR4DvxtjHOd+qzPoFX7Kjc9XVTMS1/t5tklO/j2ZCFTB3fh55MGaAlWKUdhLA1aeu4Jnvx0Gws3ZBIe1oxZ487hlrG9iWhZzcSY4kJ4ZTrs/Rpu/QS6JtRPwdJ07P4KXrwczpkA178JIaFn3Twvv5B/LEvn+WW7OFVYzNXDY7hrYn+6t9c9s0VhLA3UnkMneWbxDhas3UdYaAgzx/Tmx8nn0L51JeeEK7IW3r0L1r4I056FoTcEvmBpmlL+Ce/dDaPvhMkP+fSS3OOneWbxDl5duQeAq5NiuH18X3p01K07m7KzhbGWk5F6tzUrj78u2cG7Gw8Qagw3nteLn1zQl84RLX1v5MsnnSAed6+CWAIr6YeQvRm++gt06AMjbq32JVHhLfjd5XHcOrYPzy7ZyYKUfbyxei9XDe3OTyb05ZxOGr6W8tQzlnqzfu8Rnl28g0/SsmkdFsoNo3oya9w5RLetQQgDbHwT/vsjiL8apv/DWT1JJJCKC+GNH8C2j2HGPyHuqhq9/MDRU8z/YhevrdpDYXEJlw3pxu0T+hL7/9u78+A4z/qA49+ftJeO1S1Zso5YPhRf8RETY2KXBEgcAwZDOUIIV2AGOm1a6NALmOl0aEvKMFwzQDuUpA0pIYXmMp0kJiWEkMNX4jix41iRLVmyLFm3dqW9d5/+8axkWciOrevdtX6fmXf2ed93tfvTq9X+3ud5n/d5aormKGCVibSZWjkmlTL89ngP9zzXynMt/RTnufnM9Uv4zPVLKJ3qFqU389qj8Ms74Krr4RMPgksng1fzJBaC+z4InS/C7b+wvfcvU28wyt3PtnLfC22MxpJsW17B57Y1ckNT5fkD2KgrkiZjNe9Gogl+ebCDe59vo60/RHWRjzu2LuH2LVdNf7D944/b2kntJvjEQ+DVpj41z8KD8B/vhYETcOvPYMVN03qZoVCM+/e3c+/zbZwNRFlWWcBntzXyxxvryPNcvJOYyl6ajNW8ae0b5acvtPHLg6cZiSa4tqGEO7Y2smNtNe6ZjFB05EF46AtQvRY+9Sj49D5O5ZDRfrhvF/QetyN0rXzvtF8qlkjx2Ktd3P1sK692DlOa7+ajb6nnts0NLKkomL2YVUbQZKzmVCSeZM/Rbn6+v529Jwdw5Qg719Vwx9ZG1tdf+N7MS7b/3+Gxv4aGLXDbAxe931OpeREetMOvdh2GD/wrrPvojF7OGMOBtkHuebaVJ4+dJZkybF1ezsc3X8XNqxfhcWm/iCuBJmM1J5rPBvn5/nYePtTJUChOfVkeH7uugY9sqqPqcjtlTSWVgqe/Ac98C5rebTvOuPV+TZUhIgH4+W1w6lm48Stww9/OyuhvZwMRfnGggwcOdNA5FKai0MOHN9XzoWtrWbHIPwuBK6doMlazpns4wq8On+HRw50c6Qzgyc1h+5pF3La5gbctLZ+9TiiRADz8BTj+GGz8JOz8HuTqnXgqwySi8KsvweH7be/+XT+YtRPGZMrwTHMv9+9v56nXe0imDGtri/jgxjret77m8m4FVBlBk7GakeFwnCeOdPHIoTPsbe3HGFhfV8yuDbV8YGPt1BM3zERvM/z37dB/AnbcBZs/r+NNq8xlDDz7HfjN12HRWnsdeZYnK+kNRtl9+AyPHOrk1c5hcnOEbcsr+ODGWt61qurNR6tTGUGTsbpsfSNRfnPsLHuOnuXZN/qIJVM0VhSwa8Ni3r9+8dwMWmAMHLwH9nwNPPnwkXuh8Y9m/32UmgvNe+DhP7G15Z3fhfW3zsnbtPQEefhQJ48cOkPnUBhPbg7bVlSwY201N69aNL1bBtW80GSsLkl7f4hfv9bNnqPdHDw1iDFQV5rHLWuq2bVhMdfUFl945qSZCp6F3X8Ob+yx92/u+hEU1czNeyk1V4Y74cHPQfsLsOp98J5vg3/RnLxVKmV4qX2Qx49088SRbjqHwuTmCFuWlrFjTTXvWFlFXakOv5lJNBmrKUXiSfae7OeZ5j6eeaOXlp4RAFZW+7llTTXb1yxidU3R3CVggFQSDtwNT/2jrVHc/HXbLK2jaqlslUzA89+Hp79prx/f8g1Yf9ucfqaNMRzpDPDE0S4eP9LNyd5RAJZXFXJjUyU3Xl3FdY2leF16D7OTNBkrwJ5JN/cE+X06+e5rHSCWSOFx5fDWxjJuaKpk++pqGsrn6Wz61PPwxFeg62U7K857vg0Vy+fnvZWaa73NsPtO6NhnB6q55S5oeOu8vHVLzwhPH+/hd8297Ds5QCyZIs+dy/XLynl7UyVblpazoqpQR/2aZ5qMF6hEMsWxriD7WvvZ1zrAgbYBhkJxwJ4xv31FJW9vquCtjeXzO+pP9xHb2eWNPVBYDbf8M6z9kHbSUleeVApeecB+3oNddkzrG78KlU3zFkIoluCFE/38rrmXp4/30j4QAqCswMPmJWVsWVrGlmXlNFX5NTnPMU3GC8RQKMYrp4d55fQQB9oGefHUICPRBABXleezeUkZ1zWWsW15BYudmF/19Iu2+e613eArgm1/CZu/YDtrKXUli43aWZ+e+z7Ew7DmA3bGsepr5j2UjoEQe0/aE/S9J/s5PRgGoCTfzcb6EjY2lLKhvoT19SUU52kv7dmkyfgKFI4lOXJmmMMdQ+MJuK0/NL7/6kV+Njfa5Lt5SRnVxQ7dk5iM216me38Ep54Db7Gdgm7rX0BeqTMxKeWU0T544Yd2VLlYEJa+w/4/NL3bsfvoOwZC7GsdYH9rP4fah2jpHWEsLSyrLGBDfSkbGkpYs7iIldV+8j16v/90aTLOYqmUoWMwxOvdQY53B3m9O8Dr3UHa+kZJpf90NcU+1teVsK6+mPV1JaytLXb+jLavBQ7dBy/fD6M9UFQHb/tTuPZT4NVRhNQCFx6E/T+xt/IFz4B/MWz6tB1Ws2ypo6EFInFe6Rjm5Y5BDrUP8XLHEP2jMcBeSWosL2BVTRGrFxexqsbPqpoiqot8c9vR8wqhyTgLROJJ2gdCtPaN0to3ysneEZrPjtB8NkgolgTsP0JDWT5XL/KzstrPunQCzpiRePpa4Nhuu5w5BJILTbfYEbRWbNcRtJSaLJmA5ifg4N1w4im7bfG1tg/F6l1QUu9sfNie2qcHwxzrCvBaV2D8sWMgPP4cv8/FsspCu1QVjJevKs+f2QQxVxhNxhkiEInTORjmzFCYjnTiPZlOvp1DYSb+KSoKPayo8nN1tU28V1f7aVrkp2C60w/OhdiovZ/y5O/gjSeh95jdXrvJfpGsuxX81c7GqFS2GOqAow/ZGcq6DtttVath+U2w4mao3wKuzBnQIxCJ83pXkGNdAVp6RjjRa5ezgej4c1w5QkN5PkvKC2goy6euNI/6snzqS/OpL8tbcCOHaTKeB5F4kt5glJ5glK7hMJ2DYTqHbOI9nS4HI4nzfsbvddFYWUBjxfnLkooCijLxQzraD2degtMHoe330LEfUnHIcdsZlVbuhFU7objO6UiVym59LXZc9pYn4dQL9v/MnW9PdBveZm+RqttsO0JmmGAkzsne0fHkfKJnlFMDIToGQuMdSseU5rvHk/OiIh81xT4WFfuoLrJLVZEXn/vKuTdak/E0hWNJBkMxu4zG6RuJ0hOM0BOI0jsSpSdg13uDUQKTEi3YppvakjzqSvOoLcljcUketelyXWk+FYWezLzOkkrB0Ck7X2vvMXuW3vmS3QaAQM06aLwBlt5gvxw8OveqUnMiGrStT22/ty1R3a+CSQEC5cvtHN/V10D1Ojs2tr86I28TNMYwFIrTMRiiYyBMx2CI9nSS7hwM0x2IjF+Sm6g0382iIh/VxT4qC72UFXqoKPBSVuChvNBDeYHdVl7gyfjEvWCTsTGGSDxFMBInGE0QjCQYiSTG1wPhOEOhOIOh2PjjYCjO4KhNwNFEasrX9bpyqCryUuX3UeX3Uun3UuW365V+L9XFPmpL8zKzdjsmEYXh0zbBDrXbZfAU9DVD3xuQOHc9iOJ6qL3WnpUvvhYWb9BOWEo5JRq0rVMd+6H7FZucx0+UAU8hlDVC2TIoX2Y7hJU2QtFi8NeAO0P6mExijCEYTXB2OEJ3IELXcGS8fDZgH/tHYvSPxIglp/5uLvDkUl7opbTAQ3Gem+I8N0U+l30cX3ef25dn9/l9bnLn4R7rGSdjEdkBfB/IBX5ijPmXSfu9wE+BTUA/cKsxpu1irzmbyfiZ5l7+8/k2RiIJApE4I2OJN5ogmbr475cjUJLvoSTfTWm+h9Kxx4Lzt5Xke6go9FJV5MXvdWVejTYZt9MORochMmx7a4702p7MIz0w2nv+40j3+T8vuVBcCxVNULkSKq+2jxVNkFfizO+klLo04SE4e9QuAyfsjGcDJ+wJtplU28wrs4l5LDkXVEJ+md2eX54ul9qyrzjjatnGGEaiCZuYR2P0j0QZGB0rx+gfteuBiK1wBcJxhsNxEm+SC3zuHAq9bgq9uRR4XRR6XZQXevjR7ZtmLfaLJeM37Q0kIrnAD4GbgdPAARHZbYx5bcLTPgcMGmOWi8jHgG8CczNlyRQi8SQ9wQh+r73+4Pe58HtdFPpc+H1uCr0uu83notDrTj+6KPLZ8qyMOmOMHWc5FYdUwibH8ce43TdWHts3cX8iagcDiIchEZlQDk/aHoJYCKIBm3QjAVuOhy4cW64HCqqgsNL+89Wsg+IGKJmw+Gu0t7NS2SqvBJZstctEyXi65esUBLogcMbeShVIL2cOQag/3ew9BckBj99ehvIW2lq3t9BuG1v3FNjr2S4PuHzg8kKu91x5fPHZ7yKX1/YzycmFHNekJecPt006GRAR/D5bm11ScWmXx4wxhGJJAhGbmAPhRPrRrgcjCUZjtgI3EkkwGrXl4XB8On+NabmUb9/NQIsx5iSAiDwA7AImJuNdwD+ky/8D/EBExMxTG/h21yG2533bfqAiBsKp9IfL2CRpjF0f35aaYtuk7UzYf962KX7epGxCnQs5bjvYvDvPfpjd+bbsK7Zntt4iWx5bvEW2U4evBAqr7FlvBp7dKqXmQa7bNlWXL7vwc1Ip26IWGrBLeMAm6LFybBSiI3aQkuiIXQ+dss3lsVGIjdiKwlyRCQlacm0Sl5z0IoCcK0vOhHXGyyJCgeRQgFAz5c9NWkfsiYHHDzwyd7/bBJeSjGuBjgnrp4HJo52PP8cYkxCRYaAc6Jv4JBH5PPB5gIaGhmmGPIUcd/oappz/Rxr/w1zojzVh23nPnbztEp6X47Yf/BzXucfx8ti+3Allt62J5rhs2eWdlHB94MrT2qpSam7l5Nhm6bzSiyftizEGkjGblBPRc0sy+ofbEhHbdJ5KnmshTCWmWJ9qW3p9yoqT4Q8qYEwuT1W5MhfYl7LfxfNkXr/pjTE/Bn4M9prxrL3wipvsopRSav6JnGuOVtNyKUOjdAITh4GpS2+b8jki4gKKsR25lFJKKfUmLiUZHwBWiEijiHiAjwG7Jz1nN/DpdPnDwFPzdb1YKaWUynZv2kydvgZ8J7AHe2vTPcaYoyLydeCgMWY3cDdwn4i0AAPYhK2UUkqpS3BJ14yNMY8Bj03a9vcTyhHgI7MbmlJKKbUw6HQaSimllMM0GSullFIO02SslFJKOUyTsVJKKeUwTcZKKaWUwzQZK6WUUg7TZKyUUko5TJOxUkop5TBNxkoppZTDxKkhpEWkFzjlyJs7p4JJ00qqy6bHcOb0GM4OPY4zt9CO4VXGmMqpdjiWjBciETlojHmL03FkMz2GM6fHcHbocZw5PYbnaDO1Ukop5TBNxkoppZTDNBnPrx87HcAVQI/hzOkxnB16HGdOj2GaXjNWSimlHKY1Y6WUUsphmowdIiJfFhEjIhVOx5JtRORbIvK6iLwiIg+LSInTMWULEdkhIsdFpEVE/s7peLKNiNSLyG9F5DUROSoiX3Q6pmwlIrkickhE/tfpWDKBJmMHiEg9sB1odzqWLPUksNYYsw5oBr7icDxZQURygR8C7wZWA7eJyGpno8o6CeDLxpjVwBbgz/QYTtsXgWNOB5EpNBk747vA3wB6wX4ajDG/NsYk0qt7gTon48kim4EWY8xJY0wMeADY5XBMWcUY02WMeSldDmKTSa2zUWUfEakD3gv8xOlYMoUm43kmIruATmPMYadjuUJ8Fnjc6SCyRC3QMWH9NJpIpk1ElgAbgX3ORpKVvoetkKScDiRTuJwO4EokIv8HVE+x62vAV7FN1OoiLnYMjTGPpp/zNWyz4c/mMzalRKQQeBD4kjEm4HQ82UREdgI9xpgXReRGp+PJFJqM54Ax5qaptovINUAjcFhEwDavviQim40x3fMYYsa70DEcIyKfAXYC7zJ6f96l6gTqJ6zXpbepyyAibmwi/pkx5iGn48lCW4H3i8h7AB9QJCL/ZYz5hMNxOUrvM3aQiLQBbzHGLKSB0mdMRHYA3wFuMMb0Oh1PthARF7bD27uwSfgA8HFjzFFHA8siYs+i7wUGjDFfcjqebJeuGf+VMWan07E4Ta8Zq2z0A8APPCkiL4vIvzkdUDZId3q7E9iD7Xj0C03El20r8EngnenP3svpGp5SM6I1Y6WUUsphWjNWSimlHKbJWCmllHKYJmOllFLKYZqMlVJKKYdpMlZKKaUcpslYKaWUcpgmY6WUUsphmoyVWiBE5Lr0HNA+ESlIz8e71um4lFI66IdSC4qI/BN2POA84LQx5i6HQ1JKoclYqQVFRDzYMakjwPXGmKTDISml0GZqpRaacqAQO7a3z+FYlFJpWjNWagERkd3AA9ipPGuMMXc6HJJSCp3PWKkFQ0Q+BcSNMfeLSC7wvIi80xjzlNOxKbXQac1YKaWUcpheM1ZKKaUcpslYKaWUcpgmY6WUUsphmoyVUkoph2kyVkoppRymyVgppZRymCZjpZRSymGajJVSSimH/T9Z9kUUooQPBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "a = np.arange(-5, 5, 0.01)\n",
    "plt.figure(figsize=(8,6))\n",
    "plt.plot(a, sigmoid(a) * (1-(sigmoid(a))))\n",
    "plt.plot(a, 1 - (np.tanh(a) **2))\n",
    "plt.legend(['Derivative of $sigmoid(x)$', \n",
    "            'Derivative of $Tanh(x)$'])\n",
    "plt.title(\"Derivatives of $sigmoid(x)$ and $Tanh(x)$\")\n",
    "plt.xlabel(\"x\");\n",
    "# plt.savefig(IMG_FILEPATH + \"06_sigmoid_tanh_derivative.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross entropy/softmax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\text{CE}(p_i, y_i) = - y_i * \\text{log}(p_i) - (1 - y_i) * \\text{log}(1-p_i) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ CE(x, 0) = \\sum_{i}{(- y_i * log(p_i) - (1 - y_i) * log(1-p_i))} $$\n",
    "\n",
    "$$ \\begin{align} = &\\sum_{i}{(- 0 * log(p_i) - (1 - 0) * log(1-p_i))} \\\\\n",
    "=  &\\sum_{i}{- log(1-p_i)} \\end{align}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ CE(x, 1) = \\sum_{i}{(- y_i * log(p_i) - (1 - y_i) * log(1-p_i))} $$\n",
    "\n",
    "$$ \\begin{align} = &\\sum_{i}{(- 1 * log(p_i) - (1 - 1) * log(1-p_i))} \\\\\n",
    "=  &\\sum_{i}{- log(p_i)} \\end{align}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\text{Normalize}(\\begin{bmatrix} x_1 \\\\ x_2 \\\\ x_3 \\end{bmatrix})  = \\begin{bmatrix} \\frac{x_1}{x_1 + x_2 + x_3} \\\\ \n",
    "\\frac{x_2}{x_1 + x_2 + x_3} \\\\\n",
    "\\frac{x_3}{x_1 + x_2 + x_3}\n",
    "\\end{bmatrix} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\text{Softmax}(\\begin{bmatrix} x_1 \\\\ x_2 \\\\ x_3 \\end{bmatrix})  = \\begin{bmatrix} \\frac{e^{x_1}}{e^{x_1} + e^{x_2} + e^{x_3}} \\\\ \n",
    "\\frac{e^{x_2}}{e^{x_1} + e^{x_2} + e^{x_3}} \\\\\n",
    "\\frac{e^{x_3}}{e^{x_1} + e^{x_2} + e^{x_3}}\n",
    "\\end{bmatrix} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ S(\\begin{bmatrix} x_1 \\\\ x_2 \\\\ x_3 \\\\ \\vdots \\\\ x_n \\end{bmatrix})  = \\begin{bmatrix} \\frac{e^{x_1}}{e^{x_1} + e^{x_2} + e^{x_3} + \\ldots + e^{x_n}} \\\\ \n",
    "\\frac{e^{x_2}}{e^{x_1} + e^{x_2} + e^{x_3} + \\ldots + e^{x_n}} \\\\\n",
    "\\frac{e^{x_3}}{e^{x_1} + e^{x_2} + e^{x_3} + \\ldots + e^{x_n}} \\\\\n",
    "\\ldots \\\\\n",
    "\\frac{e^{x_n}}{e^{x_1} + e^{x_2} + e^{x_3} + \\ldots + e^{x_n}}\n",
    "\\end{bmatrix} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ {SCE}_1 = - y_1 * log(\\frac{e^{x_1}}{e^{x_1} + e^{x_2} + e^{x_3}}) - (1 - y_1) * log(1-\\frac{e^{x_1}}{e^{x_1} + e^{x_2} + e^{x_3}}) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\text{softmax}(\\begin{bmatrix} p_1 \\\\ p_2 \\\\ p_3 \\end{bmatrix}) - \\begin{bmatrix} y_1 \\\\ y_2 \\\\ y_3 \\end{bmatrix} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot of log loss when $y_i = 0$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEcCAYAAADN+K/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU1d348c83C9kTAgkhIYSwBkhIIKwKCIIrbri1KmjRWh9ta2tbt7burX3wqd2sfbRaF+qvWh8XrGvVqoCIiIQ17AQCBLKH7Pvk/P64NyHELEPIzCST7/v1uq+Zu8y93zuZfOfMueeeI8YYlFJKeR8fTweglFLKNTTBK6WUl9IEr5RSXkoTvFJKeSlN8Eop5aU0wSullJfSBK+UUl5KE7xSSnkpTfCqzxKRbBE5x9NxeCN9b72DJng3EJHrRGSjiFSKSK6IfCAiczwd16nSf/qeZb+f9SIS1Wb5ZhExIpJoz88RkXUiUiYiJSLyhYhMb7OfGvvz1Tw96d6z6V1EZJCIrBSRKhE5JCLXeTomT9AE72Ii8lPgj8BvgBggAfhf4LJ2tvVzb3Q9q6/H7yEHgWubZ0RkEhDcaj4ceBf4MzAIGAY8DNS12c8lxpjQVtMPXR557/YXoB7rf24J8JSIJHs2JA8wxujkogmIACqBqzvZJhu4B9iG9U/rB0wAVgGlwA7g0lbb3wMcBSqAPcDCzpa3c7w44A2gECu5/KideO604ykDXgUCgZeAJqDGPqe7uxl/NvBzYCdwHHgBCLTX3QW80SaeJ4A/dfLenWM/7/CYp/u+2du83mbZn4AnTuW97yD++4CvWy17HPglYIBEYBpQ6sR+znHieDcC77Sa3we81mr+CDC5s8+BM5+jrl7bJqZQwAHEtlqWAuQCYd38vwvBSu7jWi17CVjuiTzgycnjAXjzBFwANAJ+nWyTDWwBhgNBgD+wH/gFMABYYCeOJHs6AsTZr00ERne0vJ1j+QAZwAP2vkcBB4Dz28Szwf4HHgTsAm5tte6c7sbfavtMe/tBwBfAr+11sUAVMNCe9wMKgKmdvHfnOHHM033fRgDVzQkH8LUT0Cxn99FF/HuwvqB8gRz7eM0JPhwoBlYAFwKRHe3HieONwvoC9LH/voeAnFbrjgM+TnwOOv0cdfbaDuLaAVzUav5d4PY227xrx97e9G6bbacA1W2W3UmrL7f+MmkVjWsNBoqMMY1dbPeEMeaIMaYGK2mEYpU26o0xn2J9uK/FKukEABNFxN8Yk22MyepkeVvTgWhjzCP2vg8AzwLXtBPPMWNMCfAOMLmH4m/2pL19CfBo8zpjTC6wBrja3u4CrPcvo4vjd3XM03rfjDGHgE3A5faiBVgJZL2z++jCS8ANwLlYyfBoq2OXA3OwEv6zQKGIvC0iMW328ZaIlLaavtfOeRzA+uKbDJwFfAgcE5HxwDzgc2NMU6uXdPQ5cOZzdCqfoa+BdAAROQuYCPy1TewXG2MGdjBd3GZ/oUB5m2VlQFgnMXglTfCuVQxEOVE3faTV8zjgSJt/tEPAMGPMfuAO4CGgQET+KSJxHS1v5zgjgLjWiQCr1Ns2WeS1el6N9Q9z2vF3sP0h+zXNVgBL7edLsZJfVzo9Zg+8bwAvc+IL4zp7vsN9OxFzay/Z+1wG/L3tSmPMLmPMMmNMPFb1RRzWdZ3WFrdJes92cKzVwHysBL8aq1prnj2tbrNtR58DZz5Hp/IZaknwwP8A9xtj6jvZviuVWL98WgvH+nLrVzTBu9aXWPXSi7vYrnWn/MeA4SLS+m+TgF2qM8a8bIyZw4mf8Y91tryNI8DBNokgzBizyMnz6WjwAKfjtw1vs+5Yq/m3gFQRSQEuBv7hRFxdHvM03zeA14D5IhKPVZJ/uat9O8v+hXAQWAS82cW2u4EXsRJ9dzQn+Ln289V0nOA7crqfo7a+BtJF5Eqs6z0vt93AbnlW2cH0QZvN9wJ+IjK21bI0rKqgfkUTvAsZY8qw6in/IiKLRSRYRPxF5EIR+Z8OXvYVVonnbnvb+cAlwD9FJElEFohIAFCLdcGzqaPl7ex7A1AhIveISJCI+IpISusmd13Ix6pv7UyH8bfa5gciEi8ig7AuKL7avMIYUwu8jvVPvsEYc9iJuDo9Zg+8bxhjCrFKuy9gJbddne3biZjb+i6wwBhT1XqhiIwXkZ/ZXyyIyHCsXxLru3EMsJL42UCQMSYH+ByrKmwwsNnJfZzu56itrcBQ4HfAz40x3yhIGGMuNCe3Emo9Xdhm2yqsL8pHRCRERGZjtVpz5tegV9EE72LGmN8BP8VqLVGIVfr5IVZJtb3t67GS04VAEVaTyhvsklsAsNxengcMwWqR0tHytvt2YJWKJ2OVGIuAv2G19nHGfwP32T/L7+xG/M1eBj7CujCXBfy6zW5WAJNw8h/SiWOe1vvWJu5zOLmE2eE+7FLnL5w8hyxjzMZ2VlUAM4GvRKQKK7FnAj9rs907bUq1Kzs4zl6sKozP7flyrL/DF/bnw5lYT/dz1HZ/dcB2INsY07Y03l3fx7roXwC8AtxmjOl3JXhp58tSKZcRkWzgZmPMfzrZJgHYDQy1E5DyYiIyAKsV1LfsC9eqh2gJXvUqdj36T4F/anLvNx7E+gWhyb2H6Z2HqtcQkRCsev5DWPXCyouJSDrwGdYNUZd3sbnqBq2iUUopL6VVNEop5aV6VRVNVFSUSUxM9HQYSinVZ2RkZBQZY6LbW9erEnxiYiIbN7bXUkwppVR7RORQR+u0ikYppbyUJnillPJSmuCVUspL9ao6+PY0NDSQk5NDbW2tp0NRvURgYCDx8fH4+/t7OhSlerVen+BzcnIICwsjMTEREfF0OMrDjDEUFxeTk5PDyJEjPR2OUr1ar6+iqa2tZfDgwZrcFQAiwuDBg/UXnVJO6PUJHtDkrk6inwelnNMnErxSSnmrj3fm8/TqUx3l0Tma4J2Ql5fHNddcw+jRo5k6dSqLFi1i7969ng6rXb/5zW/cfsxly5bx+uuvu/24SnmDDzJz+fu6bJfsWxN8F4wxXH755cyfP5+srCwyMjL47//+b/Lz80/arrGxq3G13aOjBG+MoampOwMNKaVcqaC8jpiIQJfsWxN8Fz777DP8/f259dZbW5alpaUxd+5cVq1axdy5c7n00kuZOHEiAL///e9JSUkhJSWFP/7RGhe5qqqKiy66iLS0NFJSUnj1VWuEunvvvZeJEyeSmprKnXd+c4CkqqoqbrrpJmbMmMGUKVP417/+BcCLL77IFVdcwQUXXMDYsWO5++67W/ZXU1PD5MmTWbJkCdnZ2SQlJXHDDTeQkpLCkSNHeOWVV5g0aRIpKSncc889LccKDQ3lJz/5CcnJySxcuJDCwkKysrJIT09v2Wbfvn0nzbfnk08+YcqUKUyaNImbbrqJurq6Ds/1tddeIyUlhbS0NM4666xT+8Mo5SXyymuJCXNNgu/1zSRbe/idHew81rNjQEyMC+fBS5I7XJ+ZmcnUqVM7XL9p0yYyMzMZOXIkGRkZvPDCC3z11VcYY5g5cybz5s3jwIEDxMXF8d577wFQVlZGcXExK1euZPfu3YgIpaWl39j3o48+yoIFC3j++ecpLS1lxowZnHPOOQBs2bKFzZs3ExAQQFJSErfffjvLly/nySefZMuWLQBkZ2ezb98+VqxYwaxZszh27Bj33HMPGRkZREZGct555/HWW2+xePFiqqqqmDZtGn/4wx945JFHePjhh3nyySeJiIhgy5YtTJ48mRdeeIEbb7yxw/eitraWZcuW8cknnzBu3DhuuOEGnnrqKa6//vp2z/WRRx7hww8/ZNiwYe2ev1L9QX5ZLXPGRLlk31qCP00zZsxoaY+9du1aLr/8ckJCQggNDeWKK67g888/Z9KkSXz88cfcc889fP7550RERBAREUFgYCDf/e53efPNNwkODv7Gvj/66COWL1/O5MmTmT9/PrW1tRw+bI1BvXDhwpZ9TJw4kUOH2u9vaMSIEcyaNQuAr7/+mvnz5xMdHY2fnx9LlixhzZo1APj4+PDtb38bgKVLl7J27VoAbr75Zl544QUcDgevvvoq1113XYfvxZ49exg5ciTjxo0D4Dvf+Q5r1qzp8Fxnz57NsmXLePbZZ3E4nBoOVCmvUlXXSEVdIzHhWoLvtKTtKsnJyZ1eQAwJCelyH+PGjWPTpk28//773HfffSxcuJAHHniADRs28Mknn/D666/z5JNP8umnn570OmMMb7zxBklJSSct/+qrrwgICGiZ9/X17fAagDPxtae5KeKVV17Jww8/zIIFC5g6dSqDBw8+5X35+fm1e65PP/00X331Fe+99x5Tp04lIyOjW/tXqq/KL7fu5xgaEdDFlt2jJfguLFiwgLq6Op555pmWZdu2bePzzz//xrZz587lrbfeorq6mqqqKlauXMncuXM5duwYwcHBLF26lLvuuotNmzZRWVlJWVkZixYt4g9/+ANbt279xv7OP/98/vznP9M86tbmzZu7jNff35+GhoZ2182YMYPVq1dTVFSEw+HglVdeYd68eQA0NTW1fJG9/PLLzJkzB7C6BTj//PO57bbbOq2eAUhKSiI7O5v9+/cD8NJLLzFv3rwOzzUrK4uZM2fyyCOPEB0dzZEjR7o8P6W8SZ6d4LUO3kNEhJUrV3LHHXfw2GOPERgYSGJiIn/84x85evToSdump6ezbNkyZsyYAVjVG1OmTOHDDz/krrvuwsfHB39/f5566ikqKiq47LLLqK2txRjD73//+28c+/777+eOO+4gNTWVpqYmRo4cybvvvttpvLfccgupqamkp6fz6KOPnrQuNjaW5cuXc/bZZ2OM4aKLLuKyyy4DrJL+hg0b+PWvf82QIUNaLgQDLFmyhJUrV3Leeed1euzAwEBeeOEFrr76ahobG5k+fTq33norJSUl7Z7rXXfdxb59+zDGsHDhQtLS0jrdv1LeprkE76pWNL1qTNZp06aZtgN+7Nq1iwkTJngoov4jNDSUysrKdtc9/vjjlJWV8atf/crNUXVMPxfKGzy9OovlH+wm8+HzCQ3oXnlbRDKMMdPaW6cleNWpyy+/nKysrG9cH1BKnb68slpCA/y6ndy7ogleAXRYel+5cqWbI1Gq/8gvryUm3DUXWEEvsiqllMfkldcy1EX176AJXimlPKagvM5lbeBBE7xSSnlEU5Oxq2g0wSullFcprqqnsckwVBO8Z4kIS5cubZlvbGwkOjqaiy++2INRec78+fNp25xVKXVqWtrAa4L3rJCQEDIzM6mpqQHg448/ZtiwYR6Oqme5q7vjtsdx9ri9pTtmpXrKiW4KNMF73KJFi1p6g3zllVe49tprW9Z11K1vdnY2c+fOJT09nfT0dNatWwfAqlWrmD9/PldddRXjx49nyZIltHfD2RNPPNHSxe4111wDQHFxMeeddx7JycncfPPNjBgxgqKiIrKzs0lJSWl57eOPP85DDz0EwLPPPsv06dNJS0vjyiuvpLq6GrAG6rj11luZOXMmd999d4fnUVNTwzXXXMOECRO4/PLLW77o2srIyGDevHlMnTqV888/n9zcXMAq8d9xxx1MmzaNP/3pT984bklJCYsXLyY1NZVZs2axbds2AB566CGuv/56Zs+ezfXXX9+9P5xSvVRLNwUubCbZt9rBf3Av5G3v2X0OnQQXLu9ys2uuuYZHHnmEiy++mG3btnHTTTe19EfTUbe+Q4YM4eOPPyYwMJB9+/Zx7bXXtlRtbN68mR07dhAXF8fs2bP54osvWvp/abZ8+XIOHjxIQEBAS3e6Dz/8MHPmzOGBBx7gvffe47nnnusy9iuuuILvfe97ANx3330899xz3H777QDk5OSwbt06fH19+cUvftHuefz1r38lODiYXbt2sW3btnb7hG9oaOD222/nX//6F9HR0bz66qv88pe/5Pnnnwegvr6+5dyXLVt20nFvv/12pkyZwltvvcWnn37KDTfc0NLl8c6dO1m7di1BQUFdnqdSfUl+WS0+AtGhmuA9LjU1lezsbF555RUWLVp00rqPPvqIt99+m8cffxygpVvfuLg4fvjDH7JlyxZ8fX1PGuZvxowZxMfHAzB58mSys7O/keBTU1NZsmQJixcvZvHixQCsWbOGN998E4CLLrqIyMjILmPPzMzkvvvuo7S0lMrKSs4///yWdVdffTW+vr6dnseaNWv40Y9+1BJTamrqN46xZ88eMjMzOffccwFwOBzExsa2rG/uiri9465du5Y33ngDsDp3Ky4uprzc6vf/0ksv1eSuvFJ+eR1RoQH4+bquIqVvJXgnStqudOmll3LnnXeyatUqiouLW5Z31K3vQw89RExMDFu3bqWpqYnAwBN1bc509/vee++xZs0a3nnnHR599FG2b+/414ufn99JQ/LV1ta2PF+2bBlvvfUWaWlpvPjii6xataplXevuhDs6D2cYY0hOTubLL79sd33bboud7ca4u90dK9Xb5bm4iSRoHfwpuemmm3jwwQeZNGnSScs76ta3rKyM2NhYfHx8eOmll05pUIumpiaOHDnC2WefzWOPPUZZWRmVlZWcddZZvPzyywB88MEHHD9+HICYmBgKCgooLi6mrq7upF4nKyoqiI2NpaGhgX/84x8dHrOj82h9zMzMzJY68taSkpIoLCxsSfANDQ3s2LHDqXOdO3duS1yrVq0iKiqK8PBwp16rVF/l6jbwoAn+lMTHx7dUVbR2//3309DQQGpqKsnJydx///0AfP/732fFihWkpaWxe/fuUyqNOhwOli5dyqRJk5gyZQo/+tGPGDhwIA8++CBr1qwhOTmZN998k4SEBMDqB/6BBx5gxowZnHvuuYwfP75lX7/61a+YOXMms2fPPmm5s+dx2223UVlZyYQJE3jggQfaHcJwwIABvP7669xzzz2kpaUxefLklovKXXnooYfIyMggNTWVe++9lxUrVjj9PinVV+WX17psoI9mLu8uWER8gY3AUWNMpw3HtbvgU5eYmMjGjRuJinLNmI69lX4uVF9W2+Bg/P3/5s7zxvHDBWNPa1+ddRfsjhL8j4FdbjiOUkr1CQXldQAM6ctVNCISD1wE/M2Vx+nPsrOz+13pXam+rrkNvCu7KQDXl+D/CNwNNHW0gYjcIiIbRWRjYWFhu9v0plGnlOfp50H1de64ixVcmOBF5GKgwBiT0dl2xphnjDHTjDHToqOjv7E+MDCQ4uJi/adWgJXci4uLT2pyqlRf445+aMC17eBnA5eKyCIgEAgXkf9njFnaxetOEh8fT05ODh2V7lX/ExgY2HKTmFJ9UV5ZLYH+PoQHuvZWJJft3Rjzc+DnACIyH7jzVJM7WM3/Ro4c2cPRKaWU5+SV1zI0PBARcelxtB28Ukq5WXZxFcMHBbv8OG5J8MaYVV21gVdKqf6g0dHE3vxKJsS6/m5tLcErpZQbZRdXUd/YxPihYS4/liZ4pZRyo125FQAkaYJXSinvsievAl8fYcyQUJcfSxO8Ukq50e68ckZHhxDg5+vyY2mCV0opN9qVW0HSUPd0h60JXiml3KS8toGjpTVuucAKmuCVUspt9uZZF1gnxGqCV0opr7Irr7kFjVbRKKWUV9mdW05YoB9xLu5FspkmeKWUcpM9eRVMGBru8j5ommmCV0opNzDGsDuvwi03ODXTBK+UUm6Qc7yGyrpGxrvpAitogldKKbfYY19gHe+mC6ygCV4ppdxid1454J4+aJppgldKKTfYlVfB8EFBhAa4dhSn1jTBK6WUG2QeLWOCG6tnQBO8Ukq5XG5ZDYeKq5kxcpBbj6sJXimlXGz9gWIAZo0a7NbjaoJXSikXW59VQkSQPxPdMExfa5rglVLKxb48UMyMkYPw8XHPHazNNMErpZQLHS2t4XBJtdurZ0ATvFJKudT6LKv+/QxN8Eop5V3WHygmIsjfbYN8tKYJXimlXGj9wWJmeqD+HTTBK6WUy+Qcr+ZISQ1njHZ/9QxogldKKZdZf6AEcH/792aa4JVSykW+zComMtifpBj317+DJnillHIJYwxfZhV5pP17M03wSinlAjuOlXOsrJYF44d4LAZN8Eop5QIfZObi6yOcO3Gox2LQBK+UUj3MGMMHmXnMHDmIQSEDPBaHJnillOph+woqOVBYxYUpniu9gyZ4pZTqcR9sz0MEzk/WBK+UUl7lg8xcpiZEMiQ80KNxuCzBi0igiGwQka0iskNEHnbVsZRSqrfILqpid14FF3i4egbAlaO/1gELjDGVIuIPrBWRD4wx6114TKWU8qgPMvMAvDvBG2MMUGnP+tuTcdXxlFKqN/h3Zi6p8RHERwZ7OhTX1sGLiK+IbAEKgI+NMV+1s80tIrJRRDYWFha6MhyllHKpA4WVbM0pY9GkWE+HArg4wRtjHMaYyUA8MENEUtrZ5hljzDRjzLTo6GhXhqOUUi71z6+P4OcjXJE+zNOhAG5qRWOMKQU+Ay5wx/GUUsrd6hodvJ6Rw8IJQxgS5tnWM81OKcGLSKSIpDq5bbSIDLSfBwHnArtPPUSllOr9Pt6ZT0lVPdfOSPB0KC26vMgqIquAS+1tM4ACEfnCGPPTLl4aC6wQEV+sL5L/M8a8e5rxKqVUr/TKhsMMGxjE3LG9p6rZmVY0EcaYchG5Gfi7MeZBEdnW1YuMMduAKacdoVJK9XKHiqv4Yn8xPz13HL4e6hq4Pc5U0fiJSCzwLUBL4Eop1cY/vz6Cj8C3pg33dCgncSbBPwJ8CGQZY74WkVHAPteGpZRSfUODo4nXNuawYPwQhkb0jourzbqsojHGvAa81mr+AHClK4NSSqm+4u0txyiqrGPJzBGeDuUbuizBi8g4EflERDLt+VQRuc/1oSmlVO/W1GR4enUWSTFhzE/qPRdXmzlTRfMs8HOgAVounl7jyqCUUqov+GR3AfsKKrlt/mhEes/F1WbOJPhgY8yGNssaXRGMUkr1FcYY/nfVfuIjg7g4tXd0TdCWMwm+SERGY3cUJiJXAbkujUoppXq5DQdL2Hy4lP86axR+vr1zaA1n2sH/AHgGGC8iR4GDwFKXRqWUUr3cU6uzGBwygKt7WdPI1pxpRXMAOEdEQgAfY0yF68NSSqneK/NoGav2FHLX+UkE+vt6OpwOOdNVwQNt5gEwxjziopiUUqpXW/7BbiKD/bn+jN7XNLI1ZyqOqlpNDuBCINGFMSmlVK/1+b5C1u4v4ocLxhIe6O/pcDrlTBXN71rPi8jjWHe2KqVUv9LUZFj+wW7iI4NYOqv39BrZke5c+g3GGsBDKaX6lbe3HmPHsXLuPC+JAL/eW/fezJk6+O2cGEvVF4jG6p9GKaX6jbpGB49/tIfkuHAuTYvzdDhOcaaZ5MWtnjcC+cYYvdFJKdWvPL82m5zjNfz3FZPw6UVdAnemwwQvIoPsp22bRYaLCMaYEteFpZRSvUfO8Wqe+GQf502M6VUDenSlsxJ8BlbVTHtfVQYY5ZKIlFKql3no7Z0APHhpsocjOTUdJnhjzEh3BqKUUr3RRzvy+M+ufH5+4XiGDQzydDinxJk6eEQkEhgLtPRmb4xZ46qglFKqN6iub+Thd3aSFBPGTXP6XpnXmVY0NwM/xmoauQWYBXwJLHBtaEop5VmPf7iXo6U1vHbrGfj30g7FOuNMxD8GpgOHjDFnYw2kXerSqJRSysPWZRXx/BcH+c4ZI5ieOKjrF/RCziT4WmNMLYCIBBhjdgNJrg1LKaU8p6K2gbte28aoqBDuvXCCp8PpNmfq4HNEZCDwFvCxiBwHDrk2LKWU8pxH3tlJblkNb9x2JkEDev8dqx1xpi+ay+2nD4nIZ0AE8G+XRqWUUh7y4Y48XsvI4fYFY5iSEOnpcE6LMxdZnwD+aYxZZ4xZ7YaYlFLKI46UVHPXa1uZNCyC2xeM9XQ4p82ZOvgM4D4RyRKRx0VkmquDUkopd6trdPCDlzdhgL9cl84Av77XaqatLs/AGLPCGLMIqyXNHuAxEdnn8siUUsqNfvPeLrbllPH41WkkDA72dDg94lS+osYA44ERwG7XhKOUUu73ztZjrPjyEDfPGcn5yUM9HU6P6TLBi8j/2CX2R4DtwDRjzCUuj0wppdwg82gZd72+lakjIrnnwvGeDqdHOdNMMgs4wxhT5OpglFLKnQrKa7l5xUYGBQ/g6aVT++Tdqp1xppnkX90RiFJKuVNtg4PvvZRBWU0Dr992BtFhAZ4Oqcc51dmYUkp5k6Ymw12vb2PrkVL+ev1UkuMiPB2SS3jX7xGllOqCMYZfv7eLd7Ye494Lx3vVRdW2nLnIOlpEAuzn80XkR3bXBUop1ef8dc0Bnv/iIDfOTuS/zvLucYucKcG/AThEZAzwDDAceLmrF4nIcBH5TER2isgOEfnxacaqlFKn5Y2MHJZ/sJtL0uK4/6KJiPSNsVW7y5kE32QPsn058GdjzF1ArBOvawR+ZoyZiNWH/A9EZGL3Q1VKqe57b1sud7+xjdljBvP41al9ZuDs0+FMgm8QkWuB7wDv2sv8u3qRMSbXGLPJfl4B7AKGdTdQpZTqrg935PHjf24mPWEgz1w/jQC/vttD5KlwJsHfCJwBPGqMOSgiI4GXTuUgIpKINVDIV+2su0VENorIxsLCwlPZrVJKdenT3fn88OVNTIqP4IUbZxAS0H8aD4oxxvmNrbFZhxtjtp3Ca0KB1VhfEG92tu20adPMxo0bnY5HKaU68+GOPG5/eTPjY8N46bsziQjqsvKhzxGRDGNMu51AOtOKZpWIhIvIIGAT8KyI/N7JA/tjXaT9R1fJXSmletK/thzl+//YxMS4cF66yTuTe1ecqaKJMMaUA1cAfzfGzATO6epFYl2efg7YZYxx6gtBKaV6wj83HOaOV7cwPTGS/3fzTCKC+19yB+cSvJ+IxALf4sRFVmfMBq4HFojIFnta1J0glVLKGcYY/vLZfu59czvzxkXz4o0zCO1Hde5tOXPmjwAfAl8YY74WkVFAl/3BG2PWAt7fDkkp1Ss4mgwPvb2Dl9Yf4rLJcfz2qjSvGLTjdDjT2dhrwGut5g8AV7oyKKWUOhU19Q7ueHUzH+7I57/OGsU9F4zvF+3cu+LMRdZ4EV5yVRQAABvJSURBVFkpIgX29IaIxLsjOKWU6kp+eS3ffuZLPtqZzwMXT+TniyZocrc58/vlBeBtIM6e3rGXKaWUR2UeLeOyJ79gf0Elz1w/jZvmjPR0SL2KMwk+2hjzgjGm0Z5eBKJdHJdSSnXqna3HuPrpL/H1Ed647UzOnRjj6ZB6HWcushaLyFLgFXv+WqDYdSEppVTHGh1NPPbv3Tz7+UGmjYjkqaVTvXKwjp7gTIK/Cfgz8AfAAOuAZS6MSSml2lVUWcftL2/mywPFfOeMEfzyoon9vqVMZ5xpRXMIuLT1MhG5A/ijq4JSSqm2vswq5sf/3ExZTQO/uzqNK6dqW4+udPer76c9GoVSSnXA0WT403/2seRv6wkN9OOtH8zW5O6k7t7ipW2QlFIud6y0hp/+3xbWHyjh8inD+PXilH7VG+Tp6u475XwXlEop1Q3vbD3GL1dux9Fk+O1VqVw1Nd7rR2DqaR0meBGpoP1ELkCQyyJSSvVrZdUNPPTODlZuPsrk4QP50zWTGTE4xNNh9UkdJnhjTJg7A1FKqc92F3Dvm9sorqznxwvH8sMFY/D31VYy3aWVWUopjyurbuDX7+3ktYwckmLCeO4700kZFuHpsPo8TfBKKY8xxvD+9jwefHsHx6vr+cHZo/nRwrH9ZsxUV9MEr5TyiKOlNTz4rx38Z1c+KcPCWXHTdJLjtNTekzTBK6XcqsHRxHNrD/Kn/1jDSvz8wvF8d85I/LSuvcdpgldKuc26/UU89M4O9uZXcu7EGB68ZCLxkcGeDstraYJXSrnc0dIaHn1vJ+9vz2P4oCD+dsM0ztHeH11OE7xSymWq6hp5enUWz6w5gAj87NxxfO+sUQT660VUd9AEr5TqcY4mwxsZOfz2oz0UVtRxaVoc91w4nmED9R5Jd9IEr5TqMcYYPt1dwGP/3s3e/ErSEwby1+unkp4Q6enQ+iVN8EqpHpFxqITH/r2HDQdLGBkVwl+uS2fRpKHaf4wHaYJXSp2WHcfK+N1He/l0dwFRoQH86rJkrpmRoF0M9AKa4JVS3bI7r5wnPtnH+9vzCA/04+4Lklh2ZiLBAzSt9Bb6l1BKnZLdeeX8+ZP9vLc9l9AAP25fMIab544iIsjf06GpNjTBK6Wcsi2nlD9/up+Pd+a3JPbvzhnJwOABng5NdUATvFKqQ8YY1h8o4anVWazZW0h4oB93nDOWZWcmamLvAzTBK6W+wdFk+HhnHk+vPsCWI6VEhQZw9wVJXD9rBGGBWhXTV2iCV0q1qKl38HrGEf629iCHiqtJGBTMrxencNXUeL37tA/SBK+UIq+slr9/mc3LGw5TWt1A2vCB3HPBeM5PHoqvj7Zj76s0wSvVTxlj2HS4lBXrsnl/ey5NxnDexKHcNGck0xMj9QYlL6AJXql+prbBwbvbclmxLpvtR8sIC/DjhjMSuXF2IsMHade93kQTvFL9RHZRFf/46hCvZeRQWt3AmCGh/GpxCldMGUZIgKYCb6R/VaW8WH1jEx/vzOeVDYdZu78IPx/hvOQYls4awRmjBms1jJdzWYIXkeeBi4ECY0yKq46jlPqmA4WVvLrxCG9k5FBUWc+wgUH89NxxfHv6cGLCAz0dnnITV5bgXwSeBP7uwmMopWxVdY28vz2X1zbmsCG7BD8fYcH4IVw7I4GzxkVra5h+yGUJ3hizRkQSXbV/pRQ0NRk2ZJfwekYO72/PpbrewaioEO69cDxXpA9jSJiW1vuEJgf49Px9Bh6vgxeRW4BbABISEjwcjVJ9w8GiKlZuyuHNzUfJOV5DyABfLkmN41vT40lP0CaOvV6TA3K3QNZn1lRbBret7fHDeDzBG2OeAZ4BmDZtmvFwOEr1WkWVdby79Rgrtxxj65FSRGDOmCjuPC+J85OHEjRA7zTt1Y5nW8n8wGdwcA3UHLeWD02FMQvB0Qi+PZuSPZ7glVIdK69t4KMd+fxry1HWZRXjaDJMjA3nl4smcElaHEMjtAqm16ousRL5gVXWdPygtTwsDpIWwaizYdR8CI12WQia4JXqZarrG/nPrgLe3XqMVXsLqW9sYvigIG6dN4pL04aRNDTM0yGq9tRXweH1cHA1HFgNuVsBAwPCYORcmHWbldCjxoGbqtBc2UzyFWA+ECUiOcCDxpjnXHU8pfqy6vpGPt1dwPvbc/l0dwG1DU0MCQtgycwELkmLY8rwgVqv3ts01sPRDKuUfnA1HNkATQ3g4w/DZ8DZv4CR82BYOvh6pgdOV7aiudZV+1bKG1TUNrQk9dV7C6ltaCIqNICrpw7notRYpicO0qaNvYmj0SqVZ6+Bg59bpfWGKkAgNtUuoc+DhDNgQIinowW0ikYptyqqrOPjnfl8uCOPdfuLqXc0ERMewLenDefCSZrUe5UmB+Rtg+y1dkL/EurKrXXRE2DKEquEnjgbgiI9G2sHNMEr5WLZRVV8tDOPj3fms/HQcYyBhEHBfOfMEVyQEsuU4QPx0aTueY7GEwn90Bdw6EuoK7PWDR4LKVdC4hwYeRaEDvFsrE7SBK9UD3M0GbYcOc7HOwv4z6589hdUAjAxNpwfLRjL+clDmRAbpnXqnuZogGNb4NBaOLTOqnJpLqEPGg0pl0PiXBgxG8JjPRtrN2mCV6oHlNc28PneIj7Zlc9newo4Xt2An48wa9RglsxM4JwJMdoVr6c11EDORiuZH/oCcr6GhmprXdS4EyX0PpzQ29IEr1Q3GGPYm1/JZ3sK+Gx3ARsPHcfRZIgM9ufspCGcPX4I85KiCdfxSz2nusRq2XL4S2s6uslq5YJATApMuR5GnGkldBe2RfckTfBKOam8toF1+4tYvbeQVXsKyS2rBWBCbDj/ddYoFowfwpSESL1I6gnGQOlhOPKVndDXQ8FOa52PP8RNgTO+byXz4TN67UXRnqYJXqkOOJoM24+W8fneQtbsK2TT4VIcTYbQAD/mjInixwujmZcUTWxEkKdD7X8cDZC33U7o663HilxrXUA4xE+HlCusJovDpoJ///wbaYJXqpUjJdWs3V/E2n1FfJFVRGl1AwApw8K5dd4ozhobTfqISPx9fTwcaT9TXWLVmR/5yqp2OZpxov48YrhVMk+YZU1DJrqkZ8a+SBO86tdKqur5MquYtfuLWJdVxKFiK2nEhAewcHwMZ42LYs6YKAaHBng40n6kyQEFuyBnAxz52krsxfusdeILQydB+g1WVcvwmRAR79l4ezFN8Kpfqaht4OvsEtbtL2ZdVjE7c61mcaEBfswaNYhlZyYyd2wUo6NDtRmju1QWWK1bcuxkfmwz1FtNSwmOshL55OusZB43BQZoayRnaYJXXq2yrpGN2SWsP1DClweKyTxahqPJMMDPh6kJkfzs3HGcOSaKtPgI/LTaxfUaaiB3GxxtTugZUHbYWufjBzHJkHatldTjp0HkSLd1zOWNNMErr1Je28DG7BK+OljCVwdK2G4ndH9fIS1+IN+fP5ozRg0mfUQkgf5aT+tSjkYo3A3HNllNFI9mQP4OMA5rfcRw6wLojO9ZCT02rd9eDHUVTfCqTyusqGtJ6F9nl7Art5wmQ0tCv23eaGaNGkz6iIEED9CPu8s0NUHJAat6pTmh5207cSE0IMLqVXHOT6zHYdMgLMazMfcD+olXfYYxhoNFVWw8dJyN2SV8nX2cg0VVAAT6+5CeEMntC8Yyc9Qg0hO0hO4yxljJPHeLdat/7hY4tvVEvy1+QVbviunfsZP5VKuqxUerwNxNE7zqteoaHWQeLWNj9nEyDllTcVU9ABFB/kxPjOSa6cOZljiIScMiGOCnCaTHNTVBSZbVTe6xzdZj7rYTydx3gFVvPukq6wJo3GSrp8UeHnpOdY/+FVSvkVdWy6bDx9l06DibDh8n82g59Y4mABIHBzMvKZrpiYOYNiKS0dGh2gNjT2uss+rMc7dZ1Su52yA/80SLFt8AO5lfCbGTTyRzvwGejVt1SBO88ojaBgc7jpWx+XApm4+UsvnQcY7Zt/4P8PMhdVgEy2YnMnVEJOkJkUSHaTv0HlVzHPIyrbtB8zOtZF642+6rBfAPsdqbT77OSuaxaRCd5LGRiVT3aIJXLtfUZDhYXMXWI6VssaddueU0OAwAwwYGkT4ikpsTIpmSMJDkOK1u6TFNDig5CPnbrYSen2k9luec2CY0xup8a+w5VlIfmmp1l6t15n2eJnjV4/LKatmaU8q2nFK2Hilja04pFbWNAIQM8CU1fiA3zx3FlOEDmZwwkCFhgR6O2EtUFVnNEAt2Wo/5O6w7QhtrrPXiC1Fjrdv5h6ZYyTxmkrZm8WKa4NVpKaqsY/vRMrbnlLEtp5RtOWUUVNQB4OsjJMWEcUlaHJPjB5I2fCBjhoRqb4unq7YcCvdA4S7I32kl9IJdUFVwYpvgwVapfNqN1mNMMkSPB3/9Mu1PNMErpzUn88ycMuvxaFlLvbkIjIoKYc6YKCbFR5AaP5DkuHBtqng66iqhaA8U7LaSecFuq5687MiJbfyDrcQ99jyImWh1tDVkojWknN4B2u9pglffYIwht6yWHcfK2XGsjMyj5WQeLSOvvLZlm1FRIUxLHERqfAQpwyJIjgsnTAe36J7qEqtEXrQHCvfaj3tOTuS+AdaoQ8NnwtRlMGSCldi1fbnqhCb4fs7RZN08tONYGTuPlbPjWDk7c8spsdubi8Do6FBmjRpkJ/IIkoeF60hFp6rJYQ1IUbwfivba0z4rkVcXndjOL9BK5AmzIHqZlcSjx0NkorYtV6dMPzH9SFVdI7vzKtiZW87OY+Xsyi1nd145tQ1WW/MBvj6MGxrKuRNiSB4WTnJcOOOHhhMSoB8TpxhjlcaL97ea9kFxljU56k5sGzTIuuCZdKHV/DBqnPUYkaAlctVj9D/XCzU1GY6W1rAzt5zduRXsyi1nV155S1/nAOGBfkyIDee6GSOYGBfOxNhwxgwJ1eaJzqg5bt2qX3zAusuzOOvEY23pie18/KwqlMFjYMw5VkIfPNZK5iGDPRe/6jc0wfdxpdX17M6rYE9ehf1Yzp68CqrqrR77RCBxcAjJceFclR7P+NhwJsaFExcRqP2dd8QYqMy32o8fP9jq8YA11RxvtbFYA04MGmUNETd4zIlp4AitVlEepZ++PqKm3sH+gkr25FewN/9EMs8vP/GzPyLIn/FDw7hqajxJQ8OZEBtG0tAw7UWxPfVVcPwQlB6yHo9nW89LDlrPm9uOA4gPhMfDoJEwcbH1OGg0DB5t1Y1rF7eql9L//F6mrtHBgcIq9uZXsC+/kr12Qj9UUo2xbvxkgJ8PY6JDOXN0FElDwxg/NIzxQ8OJCQ/QUnmzukqrFUrpEStxlx2xE/pha2p9YROsW/MjR1gl8dELrMQ9aKRVxTIwQftbUX2SJngPqW2wEvm+ggr2F1RaybyggkPF1TiarEzu6yOMjAphYlw4i6cMIykmjLExYSQODu7fow81OawqlLKcdqbD1uNJ1ShYvR5GDLeS9fiLrGQ+cISVyCMTrRuD9MtReRlN8C5WWddIVkGllcQLKtlvJ/TDJdXYeRxfH2HE4GDGRIdy0aRYxsaEMS4mlJFRIQT49bMbhRwNUJEL5blQcQzKW09Hoeyotb55VKBmA8Jg4HAIHwbx008k8+YpZIi2TlH9jib4HmCMoaCijqyCSrIKK8kqrGK/ndRb3xzk73uiRH7Z5GGMjQllzJB+ksgdDdbgypV5UJF/4rEiFyryrGRekQdVhd98rV8QhMdZ08i5VhIPj7OSeMQwaz5ooPvPSaleThP8KahtcHCouJqswkoO2Im8+bGyrrFlu5ABvowZEsqZowczeoiVxMcMCWXEIC+rWnE0QnWxlZSrCqCy+bF5yj/xWF0MmDY7EAiJtjq7CouDuHQIiz2RzMNirQQeOFCrT5TqBk3wbTQ1GXLLazlYWMXBokoOFFVxoLCKA0WV5ByvabnQCRAbEcio6BCuTB/G6CGhjI4OZVR0CEPD+2gTxIYaKxFXl9iP9lRVZF2UrCo6MV9VCDUl7e/HN8DqgjY02qoeGT4dQoda/aOEDbWeh8VY22j/4kq5TL9M8MYYSqrqOVhU1TJlF1uJPLu4quXOTrBK4yOjQ5g8PJIrpsQzKjqE0dFWtUqvvMOzyQF15VBbBjWl1o03rR9rjrc/VZec3DTwJAJBkRASZZW4o5Mgcc6J+eapOakHhGuJW6lewKUZSkQuAP4E+AJ/M8Ysd+XxWjPGUFrdwMHiKrKLqsgurrYfrYTe3D85gJ+PkDAomJFRIcweE8Wo6BBGRoUwKirUPU0PjQFHvdU2u77SauJXXwX1FVBXYc9XWom7zl5WW24n8jaPdeWdH8vHH4IHWQk7cKBVwo6dbNVhBw+21w2ynodE2c8HgY+XXyNQygu5LMGLiC/wF+BcIAf4WkTeNsbs7MnjNDUZNh85TnZRNYeKrUR+yE7i5a2SuI9A3MAgRkaFsHjyMBKjQhgVFUJiVAjxkUH4C9ZwZU2NVrJ1NIKjGEobrAuEjnprzEpH/YnnjXVW/yKNddBYaz021FjPWx6rocGeb6i2H6ugvtqar6+25psaOz7J1nwHWCXkgDBrCoyw2m4HRljLAyPsKdxK4EEDTzwGRVrdy2rpWql+wZUl+BnAfmPMAQAR+SdwGdCjCR4g8PkFpJk6pmDw97Vaq/gPEPwDwd8H/KQJPx+QJgcUO6Cw0RotvqnRam7naOCbFwBPk1+gNfkHW3c6+gedeB4UCQOCrfkBIdbkHwwDQiEg1F4W2mo+1E7qoeCnY5MqpZzjygQ/DGjVoTU5wMy2G4nILcAtAAkJCad8EB8fIWZ0GoE+DoIG+OHr4wOIdXu5ND/6Wm2gxcfqAEp8rSoHH98T877+1nMfP6uU3DzvF2DNt37uO8BO4AOsC4r+djL3C7Ca9PkFaClZKeVxHr9KaIx5BngGYNq0ad0qRkfdsKJHY1JKKW/gykbZR4Hhrebj7WVKKaXcwJUJ/mtgrIiMFJEBwDXA2y48nlJKqVZcVkVjjGkUkR8CH2I1k3zeGLPDVcdTSil1MpfWwRtj3gfed+UxlFJKtc+LOkZRSinVmiZ4pZTyUprglVLKS2mCV0opLyXG9PAt+qdBRAqBQ6fwkiigqMutvI+ed/+i592/nOp5jzDGRLe3olcl+FMlIhuNMdM8HYe76Xn3L3re/UtPnrdW0SillJfSBK+UUl6qryf4ZzwdgIfoefcvet79S4+dd5+ug1dKKdWxvl6CV0op1QFN8Eop5aV6fYIXkQtEZI+I7BeRe9tZHyAir9rrvxKRRPdH2fOcOO+fishOEdkmIp+IyAhPxOkKXZ17q+2uFBEjIl7RlM6Z8xaRb9l/9x0i8rK7Y3QFJz7rCSLymYhstj/vizwRZ08SkedFpEBEMjtYLyLyhP2ebBOR9G4dyBjTayesboazgFHAAGArMLHNNt8HnrafXwO86um43XTeZwPB9vPbvOG8nT13e7swYA2wHpjm6bjd9DcfC2wGIu35IZ6O203n/Qxwm/18IpDt6bh74LzPAtKBzA7WLwI+AASYBXzVneP09hJ8y8Ddxph6oHng7tYuA5rH7HsdWCjS5wdE7fK8jTGfGWOq7dn1WCNmeQNn/uYAvwIeA2rdGZwLOXPe3wP+Yow5DmCMKXBzjK7gzHkbINx+HgEcc2N8LmGMWQOUdLLJZcDfjWU9MFBEYk/1OL09wbc3cPewjrYxxjQCZcBgt0TnOs6cd2vfxfq29wZdnrv9c3W4MeY9dwbmYs78zccB40TkCxFZLyIXuC0613HmvB8ClopIDtb4Ere7JzSPOtUc0C6PD7qtTo+ILAWmAfM8HYs7iIgP8HtgmYdD8QQ/rGqa+Vi/2NaIyCRjTKlHo3K9a4EXjTG/E5EzgJdEJMUY0+TpwHq73l6Cd2bg7pZtRMQP6ydcsVuicx2nBiwXkXOAXwKXGmPq3BSbq3V17mFACrBKRLKx6iff9oILrc78zXOAt40xDcaYg8BerITflzlz3t8F/g/AGPMlEIjVIZc3cyoHdKW3J3hnBu5+G/iO/fwq4FNjX6Xow7o8bxGZAvwVK7l7Q11ss07P3RhTZoyJMsYkGmMSsa4/XGqM2eiZcHuMM5/1t7BK74hIFFaVzQF3BukCzpz3YWAhgIhMwErwhW6N0v3eBm6wW9PMAsqMMbmnupNeXUVjOhi4W0QeATYaY94GnsP6ybYf66LFNZ6LuGc4ed6/BUKB1+xryoeNMZd6LOge4uS5ex0nz/tD4DwR2Qk4gLuMMX3616qT5/0z4FkR+QnWBddlfb0QJyKvYH1ZR9nXFh4E/AGMMU9jXWtYBOwHqoEbu3WcPv4+KaWU6kBvr6JRSinVTZrglVLKS2mCV0opL6UJXimlvJQmeKWU8lKa4JVSyktpgldKKS+lCV71CSLiEJEtIpIpIq+JSPBp7OshEbnTfr6uk+0Gisj32yzrcPtuxBEkIqtFxLeTbQaIyBq7Gw6lTokmeNVX1BhjJhtjUoB64NbWK+1buk/582yMObOT1QOxxhtwdvtTdRPwpjHG0dEGdhe6nwDf7sHjqn5CE7zqiz4HxohIoj0S0N+BTGC4iCwVkQ12af+vzaVjEfmliOwVkbVAUvOORKSy1fMb7NFztorIS8ByYLS9r9+2s/1P7V8UmSJyh70sUUR2iciz9qhLH4lIUAfnsQT4V6v9vSLW6GQbROSQiFxkr3rL3lapU6IJXvUpdlXFhcB2e9FY4H+NMclAMFZJd7YxZjJWfy1LRGQqVh9Fk7H695jezn6TgfuABcaYNODHwL1Alv3L4a4220/F6h9kJlaPlt+zO4BrjukvdkylwJXtHG8AMMoYk91qcRpwwBgzAyuhP2gvz2wvZqW6ovV6qq8IEpEt9vPPsTqZiwMO2SPegNXj4FTga7sDtiCgABgErGweAUtE2uuwbAHwmjGmCMAYUyIi4e1s12yOvc8qe59vAnOxegE8aIxpjjUDSGzn9VFYyR/79YFANPCwvWgnEGnH4hCRehEJM8ZUdBKTUifRBK/6ihq7VN7CTuJVrRcBK4wxP2+z3R2uD+8krfvmd2B90bRVg9XtbbMUYJ8xpnkIwnSs8UmbBeA9wxMqN9EqGuVNPgGuEpEhACIySERGYA3OvdhutRIGXNLOaz8FrhaRwc2vBSqwBhhpz+f2PoNFJAS43F7mFHtcVV+75A5W9UyCiATa+3sY+IMdy2CgyBjT4Oz+lQJN8MqLGGN2YtWjfyQi24CPgVhjzCbgVawS8QdYg0y0fe0O4FFgtYhsBX5v97X+hX0R9bdttt8EvAhsAL4C/maM2XyKIX+EVdUDVoJ/097X18BTxpgv7HVnA940/qxyE+0PXikPEWvw8J8YY64XkdXALcaYPe1s9yZwrzFmr9uDVH2aluCV8hD7V8BndlPO0cC+ttvYrW3e0uSuukNL8Eop5aW0BK+UUl5KE7xSSnkpTfBKKeWlNMErpZSX0gSvlFJeShO8Ukp5KU3wSinlpf4/BnIBU0oeEfQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0.01, 0.99, 99)\n",
    "y1 = -1.0 * np.log(1 - x)\n",
    "y2 = (x - 0) ** 2\n",
    "plt.plot(x, y1);\n",
    "plt.plot(x, y2);\n",
    "plt.legend(['Cross entropy loss', 'Mean squared error'])\n",
    "\n",
    "plt.title(\"Cross entropy loss vs. MSE when $y = 0$\")\n",
    "plt.xlabel(\"Prediction ($p$)\")\n",
    "plt.ylabel(\"Loss values\");\n",
    "# plt.savefig(IMG_FILEPATH + \"04_Log_loss_vs_MSE_y_eq_0.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When $y=0$, loss can become theoretically infinite as $p$ approaches 1."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot of loss when $y_i = 1$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEcCAYAAADN+K/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hcZ5n+8e+jbnWr27JkuceJE8eO4ziNVEIIaSwQSIMsWbJkCYSlw7I/ll12NwsLy7KhmQRCS9lASJyQEANpTnFLXOLeIjfJluSi4qry/P6YkVC8LmNbM2d05v5c17k0OnPmnOfI8q0z73nnfc3dERGR8EkLugAREYkPBbyISEgp4EVEQkoBLyISUgp4EZGQUsCLiISUAl5EJKQU8CIiIaWAl0HPzOrN7PIEHWuCmS02s3Yz+1QijilyohTwEleJDN8E+QLwvLsXuPv3gi7mSMzsLjNbaGYHzOyBoOuRYCjgRY7PSGD5ibzQzDIGuJajaQC+Afw0gceUJKOAl0CY2UQze8HMdpvZcjO79pDnp5rZomhTyKNm9oiZfWMA9vtFM9sa3e9qM7vsaOsPee1zwCXAvWbWYWbjYzhefXTfS4E9/UPezPLNrNvMhvVbN8nMGs2sIMYf5WG5+2Pu/jiw42T2I4ObAl4SzswygSeB2UAF8Eng12Y2Ifp8FvA74AGgBHgIeO8A7HcCcBdwtrsXAO8C6o+0/tD9u/ulwBzgLnfPB9462vH6uRF4D1Ds7l399tcBrAKm9tv2HuDf3L39kHN7KvpH5HDLU8f62UhqUsBLEGYA+cA97n7Q3Z8DniIShL3PZwDfc/dOd38MmD8A++0GsoFTzSzT3evdff1R1p/s8Xp9z903u/u+w+xjAdGAN7N3AKcCPz50I3e/2t2Lj7BcHUOtkoIU8BKE4cBmd+/pt24jUN3v+a3+9rGsN5/sft19HfBp4J+AJjN72MyGH2n9AJxHLLX3BTzwTeAf3f1gDMcWOSYFvAShAagxs/6/f7XA1ujjRqDazKzf8zUDsF/c/UF3v4DIzVIH/uNo60/2eL2HPco+FgBTzex9QA7w4OE2MrNnou3+h1ueiaFWSUEKeEmETDPL6V2AecBe4AtmlmlmFwPXAA9Ht3+NSLPJXWaWYWbXAdNjOM5R9xvtw36pmWUD+4F9QM+R1p/s8WK0BKgCvg18+ZB3LX3c/d3unn+E5d2Hbh/9ueUA6UB69GefyF48kgQU8JIITxMJzd7l/xEJwncDLcAPgA+7+yqAaBPFXwG3A7uBW4i0bR842kGirzvifom0s98TfW4bkRujXz7K+qOK4XjH5O4HgDeBencfyCvxrxL5WX+JyM9vX3SdpBDTlH0yGJjZPOBH7v6zoGsZSNEeQ+uAG9x9btD1SLjoCl6SkpldZGZV0aaGjwBnAH8Iuq44+BrwisJd4kFtcpKsJgD/C+QBG4D3u3tjsCUNHDObCjwPLCWGPv4iJ0JNNCIiIaUmGhGRkEqqJpqysjKvq6sLugwRkUHj9ddfb3H38sM9l1QBX1dXx8KFC4MuQ0Rk0DCzjUd6Tk00IiIhpYAXEQkpBbyISEgp4EVEQkoBLyISUgp4EZGQUsCLiITUoA/4nh7n3ufW8tKa5qBLERFJKoM+4NPSjB+/tIE/r9wedCkiIkll0Ac8QFVhDtvbjjoXhIhIyglFwFcW5rCtbX/QZYiIJJXQBPx2BbyIyNuEIuCrirJpaj9AT4/GthcR6RWKgK8szKG7x2nZo3Z4EZFeoQl4gO2tCngRkV6hCPiq3oBXO7yISJ9QBHzvFbx60oiI/EUoAr4sP4s0gyYFvIhIn1AEfEZ6GuUF2bqCFxHpJxQBD70fdtJNVhGRXqEKeDXRiIj8RWgCvkrDFYiIvE1oAr6yMJvdezvZ39kddCkiIkkhRAEf6SrZpHZ4EREgRAFfVaS+8CIi/YUm4PVhJxGRtwtdwKsnjYhIRGgCvjAngyGZ6WxrVcCLiECIAt7MqCzUp1lFRHqFJuCh98NO6kUjIgIhC/iqIn3YSUSkV9wD3szSzWyRmT0V72P1Tr7trqn7REQScQV/N7AyAcehsjCHg109tO7rTMThRESSWlwD3sxGAO8B7ovncXpVqS+8iEifeF/Bfxf4AtBzpA3M7A4zW2hmC5ubm0/qYJWF2QDqKikiQhwD3syuBprc/fWjbefuM919mrtPKy8vP6ljajwaEZG/iOcV/PnAtWZWDzwMXGpmv4rj8ajovYJXE42ISPwC3t2/7O4j3L0O+BDwnLvfEq/jAWRnpFOSl6WAFxEhZP3gAWqGDmHjjj1BlyEiEriEBLy7v+DuVyfiWKdUFbKysV194UUk5YXuCv6UYQXs3HOQ5g7daBWR1Ba6gJ9QVQDA6m3tAVciIhKs0AX8KVWFAKxqVMCLSGoLXcCX5GVRUZDNKl3Bi0iKC13AA5wyrJBV29qCLkNEJFDhDPiqAtY2ddDVfcQREkREQi+0AX+wq4d69YcXkRQWyoDv7UmzUjdaRSSFhTLgx1bkk55m6iopIiktlAGfnZHOmPI83WgVkZQWyoAHmBAdskBEJFWFNuBPqSpg6+59tO3X9H0ikppCHfAAa9QOLyIpKrwBPyw6ZIECXkRSVGgDfnhRDgU5GbrRKiIpK7QBb2ZMrCpk2VYFvIikptAGPMC0uqEs29rKngNdQZciIpJwoQ74c8eU0tXjLNy4K+hSREQSLtQBf9bIoWSkGXM37Ai6FBGRhAt1wOdmZTC5plgBLyIpKdQBD3Du6FKWbmmlQ+3wIpJiQh/wM0aX0t3jLKzfGXQpIiIJFfqAnzqymMx0Y+4GBbyIpJbQB3xuVgaTRxTzmtrhRSTFhD7gIdJdctnWVto18JiIpJCUCPi+dnj1hxeRFJISAT+1dmi0HV7NNCKSOlIi4IdkpTOlZiivrVfAi0jqSImAB7hoQjlLt7SyrXV/0KWIiCREygT8u06rAuDZ5dsCrkREJDFSJuDHVuQzriKfZ5Y1Bl2KiEhCpEzAA7x7UhXz39rJjo4DQZciIhJ3KRXwV04aRo/D7BXbgy5FRCTuUirgJw4rYGRpLn9YpnZ4EQm/uAW8meWY2XwzW2Jmy83s6/E61nHUxJWnVfHq+hZa9+lTrSISbvG8gj8AXOruk4EzgSvNbEYcjxeTKydV0dnt/HmlmmlEJNziFvAe0RH9NjO6eLyOF6vJI4oZVpTDM2qmEZGQi2sbvJmlm9lioAn4o7vPO8w2d5jZQjNb2NzcHM9yAEhLM66cVMWLa5pp3atmGhEJr7gGvLt3u/uZwAhguplNOsw2M919mrtPKy8vj2c5fd5/1ggOdvXwu0VbEnI8EZEgJKQXjbvvBp4HrkzE8Y7ltOFFnDGiiIcXbMY98FYjEZG4iGcvmnIzK44+HgK8E1gVr+Mdrxun17JqWzuLNu8OuhQRkbiI5xX8MOB5M1sKLCDSBv9UHI93XK6ZPJzcrHQenr8p6FJEROIinr1olrr7FHc/w90nufs/x+tYJyI/O4NrJw/nySWNmulJREIppT7Jeqgbp9eyr7ObJxY3BF2KiMiAS+mAP2NEEROHFfKQmmlEJIRSOuDNjJvOqWV5Qxvz39oZdDkiIgMqpQMe4P1TR1CSl8WPXlwfdCkiIgMq5QN+SFY6t51Xx3Ormli1rS3ockREBkzKBzzAh88dSW5WOj9+cUPQpYiIDJhjBryZ3W1mhRZxv5m9YWZXJKK4RCnOzeKm6bXMWtLA5p17gy5HRGRAxHIF/1F3bwOuAIYCtwL3xLWqANx+4SjSDO6bo6t4EQmHWALeol+vAn7p7sv7rQuNYUVDeO+Uah5esJnmds3ZKiKDXywB/7qZzSYS8M+aWQHQE9+ygnHnxWPp6nHufW5t0KWIiJy0WAL+duBLwNnuvpfIxB1/HdeqAjKqLI8PnV3Dr+dtor5lT9DliIiclFgC/lxgtbvvNrNbgK8CrfEtKzh3XzaOzPQ0/nP26qBLERE5KbEE/A+BvWY2GfgssB74RVyrClBFYQ4fu3AUTy1tZOkWDSUsIoNXLAHf5ZFZMa4D7nX37wMF8S0rWB97x2hK8rK455lVmhBERAatWAK+3cy+TKR75O/NLI1IO3xoFeRk8slLx/Lq+h08t6op6HJERE5ILAH/QeAAkf7w24jMr/qtuFaVBG4+ZyRjK/L52qzl7DvYHXQ5IiLH7ZgBHw31XwNFZnY1sN/dQ9sG3ysrI41/uW4SW3bt497n1W1SRAafWIYquAGYD3wAuAGYZ2bvj3dhyeDcMaX81ZRqZr60gXVN7UGXIyJyXGJpovkHIn3gP+LuHwamA/8Y37KSx1feM5Ehmel89fFluuEqIoNKLAGf5u797zTuiPF1oVCWn80XrjyFuRt28pvXtwRdjohIzGIJ6j+Y2bNmdpuZ3Qb8Hng6vmUll5um13J23VD++ckVbN29L+hyRERiEstN1s8DM4EzostMd/9ivAtLJmlpxrc/cCbd7nzhN0vo6VFTjYgkv5iaWtz9t+7+mejyu3gXlYxqS3P56ntO5ZV1O/jl3I1BlyMickxHDHgzazeztsMs7WaWknPb3Ti9hosnlPPvz6xkfXNH0OWIiBzVEQPe3QvcvfAwS4G7FyayyGRhZnzzfWeQk5nOJx9cxP5OfQBKRJJXyvSGGSgVhTl854bJrGhs4+tPrgi6HBGRI1LAn4BLT6nk4xeN4aH5m3h80dagyxEROSwF/An63BXjmV5Xwld+96Y+5SoiSUkBf4Iy0tP43o1TGJKZzh2/eJ3WvZ1BlyQi8jaxjEVzuN40m83sd2Y2OhFFJquqohx+eMtZbN61l7seeoOu7lBOVSsig1QsV/DfBT4PVBMZKvhzwIPAw8BP41fa4DB9VAn/ev3pzFnbwjd+vzLockRE+mTEsM217j653/czzWyxu3/RzL4Sr8IGkxvOrmHN9nbue/ktxlbkc8uMkUGXJCIS0xX8XjO7wczSossNwP7oc/rMftSXr5rIpadU8P+eWMbs5duCLkdEJKaAv5nIdH1N0eVW4BYzGwLcFcfaBpX0NOPem6Zw+ohiPvnQIhbW7wy6JBFJcbEMNrbB3a9x97Loco27r3P3fe7+8pFeZ2Y1Zva8ma0ws+VmdvfAlp58crMy+NltZ1NdPITbf76QNdvVfVJEghNLL5oR0R4zTdHlt2Y2IoZ9dwGfdfdTgRnAJ8zs1JMtONmV5GXx849OJzsjjVvvn8fGHXuCLklEUlQsTTQ/A2YBw6PLk9F1R+Xuje7+RvRxO7CSSE+c0KspyeWXt5/Dwa4ebvrJPDbv3Bt0SSKSgmIJ+HJ3/5m7d0WXB4Dy4zmImdUBU4B5h3nuDjNbaGYLm5ubj2e3SW1CVQG/vP0c2vd3ctN9c2nQRCEikmCxBPwOM7vFzNKjyy1Epu2LiZnlA78FPu3u/2eYYXef6e7T3H1aeflx/d1IepOqi/jl7eewe08nN/1krmaDEpGEiiXgPwrcAGwDGoH3A7fFsnMzyyQS7r9298dOsMZBbXJNMT+/fTo79hzkhh+9Rn2L2uRFJDFi6UWz0d2vdfdyd69w9+uB9x3rdWZmwP3ASnf/zgDUOmhNrR3KQx+bwb7Obm748WusVe8aEUmAEx1s7DMxbHM+kT7zl5rZ4uhy1Qkeb9CbVF3EI3fMAOCGH7/Gok27Aq5IRMLuRAPejrWBu7/s7ubuZ7j7mdHl6RM8XiiMqyzg0Y+fS+GQTG76yTyeW7U96JJEJMRONOA1RMEJGlmax28+fh5jK/L52C9e55EFm4IuSURC6oQm3SbSH15OUHlBNg/fMYPzx5bxxd++yTf/sIqeHv3NFJGBdaKTbscyCqUcRV52Bvd/ZBo3Tq/lBy+s585fv87eg11BlyUiIaIZnQKUmZ7Gv713Ev949an8ccV2PvCj19RXXkQGjAI+YGbG7ReM4v6PnM2mHXu55n9e5tX1LUGXJSIhoIBPEpecUsHjd51PSV4Wt94/n/vmbMBd7fIicuIU8ElkTHk+j3/ifK44tZJv/H4ld/7qDVr3aTJvETkxCvgkk5+dwQ9unspXrjqFP63cztX/M4elW3YHXZaIDEIK+CRkZtzxjjE88rfn0t3tvO+Hr3LfnA3qSikix0UBn8TOGjmUp+++kIsnVPCN36/kwz+dz/a2/cd+oYgICvikV5ybxcxbz+Lf3ns6Czfu5F3ffYln3mwMuiwRGQQU8IOAmXHTObU89ckLqRmay52/foNPPbSI3XsPBl2aiCQxBfwgMrYin8f+7jz+/vLxPP1mI+/8r5f40woNWCYih6eAH2Qy09O4+/JxPP6J8ynNy+JvfrGQux58g5aOA0GXJiJJRgE/SE2qLmLWXRfwmXeOZ/by7Vz+nRd5dOFmfThKRPoo4AexrIw0PnXZOJ6++wLGlOfz+d8s5UMz57KuSTNGiYgCPhTGVhTw6N+ey7//1ems2tbOu/97Dv/xh1UanVIkxSngQyItzbhxei3PffYirp1czQ9fWM+l//kis5Y0qNlGJEUp4EOmND+bb98wmd/eeS6l+Vl86qFFfHDmXJZtbQ26NBFJMAV8SJ01soRZd13Av753EuuaOrjm3pf53KNL9ElYkRSigA+x9DTj5nNG8sLnL+aOC0cza3EDF3/rBb4zezUdB9Q+LxJ2CvgUUJiTyZevmsifPnMRl02s4HvPreOibz7Pz1+t52BXT9DliUicKOBTSG1pLvfeNJUnPnE+4yrz+dqs5Vz67Rf4zetb6NZIlSKho4BPQZNrinnoYzN44K/Ppjg3k889uoR3ffclnlraoCGJRUJEAZ+izIyLJ1Tw5F0X8MObpwJw14OLePd/z+HpNxsV9CIhYMnUR3ratGm+cOHCoMtISd09zlNLG/jen9eyvnkP4yvz+cQlY7n6jOGkp1nQ5YnIEZjZ6+4+7bDPKeClv96gv/e5daxt6mBUWR53XjSG66dUk5WhN3wiyUYBL8etp8eZvWIb//PcOpY3tFFVmMPfXDiKG6fXkpedEXR5IhKlgJcT5u68tLaFH76wjrkbdlKYk8EtM0Zy23l1VBTmBF2eSMpTwMuAWLRpFzNf2sAflm8jI8247sxqPnr+KE4dXhh0aSIpSwEvA2rjjj3cN+ctfvP6FvZ1dnPu6FI+esEoLj2lQjdkRRJMAS9x0bq3k4cWbOLnr9bT2LqfmpIh3DpjJB+cVktRbmbQ5YmkBAW8xFVXdw+zV2zngVfqmV+/k5zMNK6bXM2t545kUnVR0OWJhJoCXhJmeUMrv5q7kccXNbCvs5vJNcXcPL2WqycPIzdLvW9EBlogAW9mPwWuBprcfVIsr1HAh0frvk4ee2MLv5q7kfXNeyjIzuD6KdV8aHoNpw3XVb3IQAkq4N8BdAC/UMCnLndnQf0uHpy3kaeXbeNgVw+nVxfxwbNruGbycIqGqK1e5GQE1kRjZnXAUwp4Adi99yCPL9rKwws2s2pbO9kZaVw5qYoPnFXDeWNKSVMPHJHjltQBb2Z3AHcA1NbWnrVx48a41SPJwd15c2srjy7cwhOLt9K2v4vhRTlcP6Wa9501gjHl+UGXKDJoJHXA96cr+NSzv7Ob2Su289gbW3hpTTM9DpNHFHH9lGqumTycsvzsoEsUSWoKeBkUmtr288TiBn63aCsrGttITzMuHFfGtZOHc8VpVeRrDByR/0MBL4PO6m3tPL54K7MWN7B19z6yM9K4fGIl10wexsUTKsjJTA+6RJGkEFQvmoeAi4EyYDvwNXe//2ivUcDLoXp6nDc27eKJxQ08s6yRlo6D5GdncPnECt5zxnAuHFemsJeUpg86SSh0dfcwd8NOnlzSwLMrtrF7b2df2F85aRgXTyhX2EvKUcBL6HR29/Dq+h08vbSxL+xzs9K5ZEIF75pUxSUTyinIUR97CT8FvIRaZ3cP8zbs5Olljcxevo2WjoNkpadx/thSrjitissmVlBRoLHrJZwU8JIyuqNt9s8u28azK7axeec+zGBKTTGXn1rJFadWMqY8HzN9qErCQQEvKcndWb29ndnLtzN7xTaWbW0DYGRpLpedUsllEys4u65Ec83KoKaAFwEaW/fx55VN/HHFdl7bsIODXT3kZ2dw4bgyLjmlgosnlKspRwYdBbzIIfYc6OKVdS08t6qJ51c3sb3tAACnVxdx8YRyLp5Qzpk1QzVDlSQ9BbzIUbg7KxrbeH5VEy+sbuaNTbvocSgakskFY8u4aHw57xhfTlWRru4l+SjgRY5D695O5qxr5sXVzby0trnv6n58ZT4XjivngnFlnDOqRBOYSFJQwIucIHdn1bZ25qxtZs7aFua9tZODXT1kpacxdWQxF4wt4/yxZZxeXURGum7WSuIp4EUGyP7ObhbU7+TltS3MWdvCisZIz5yCnAxmjC7l/DGlnDe2jHEV6oopiaGAF4mTHR0HeHX9Dl5d38LL61rYvHMfAGX5WcwYXcq5Y0o5d3Qpo8ryFPgSFwp4kQTZvHMvr0UD/7UNO/ra7ysLs5kxupQZo0s5Z1SJAl8GzNECXneJRAZQTUkuNSW53HB2De7OWy17eHX9Dua9tZNX1+/gicUNAJQXZDN9VAnnjCph+qgSxlcUaMpCGXAKeJE4MTNGl+czujyfW2aM7Av8eW/tZN6GSOj/fmkjEOmSeXbdUKbVlXB2XQmnVxfpE7Zy0hTwIgnSP/BvnF6Lu7Nl1z7mv7WT+W/tZEH9Tv60sgmA7Iw0JtcUM23kUM6uK2Fq7VCKcjU6phwftcGLJJHm9gMsrN/Jwo27WFi/k+UNbXT1RP6PjqvI56yRQ5k6cihTa4cyplzt+KKbrCKD1t6DXSzZ3MrrGyOhv2jTblr3dQKRZp0ptcVMrY0E/uSaIo2Bn4J0k1VkkMrNyoh0tRxTCkSmMNzQ0sEbG3fz+sZdvLFpFy+sbgbALHKVP6VmKGfWFjN5RDHjK/P1AawUpit4kUGudV8nizfvZvGm3SzevItFm3eze2/kKj83K51J1UWcWRMJ/Mk1RVQXD1HTTojoCl4kxIqGZHLR+HIuGl8ORIZX2LhjbyT0o8sDr9RzsLsHgNK8LM4YUcQZ0cA/Y0QxZfnZQZ6CxIkCXiRkzIy6sjzqyvK4fko1AAe7eljZ2MbSLbtZsqWVJZt388KaZnrfwA8vyuH0aOhPqi7i9OoiSvKyAjwLGQgKeJEUkBXtdjm5pphbo+v2HOhieUMk9N/c2sqbW1p5dvn2vtdUFw9hUnUhk4YXMam6iNOqCzUhyiCjgBdJUXnZGUyPfpK2V+u+TpY3tLJsaytvbm1j2da3h35FQXYk7IcXRpciRgxVm36yUsCLSJ+iIZmcN6aM88aU9a1r39/JioY23tzayoqGNpY3tPHimma6o/3zC3MymDgsEvanDi9k4rACxlUU6JO4SUABLyJHVZCTyTmjSzlndGnfuv2d3aza1s7yhlaWN7SxsrGNh+ZvYl9nNwCZ6caY8nxOHVbIxL6lgFLdzE0oBbyIHLeczHTOrCnmzJrivnXdPZGxdlY2trGisY0VDW28vK6FxxZt7dumvCA7EvZVBZwyrIAJlYWMrcjX1X6cKOBFZECkpxljK/IZW5HPNZOH963f0XGAlY3trNrWxsrGdlY2tvGz9Tv6um1mpBmjy/OYUFXIKVUFTKgsYEJVAdXFQzTC5klSwItIXJXmZ3PBuGwuGPeXdv3O7h7qW/awcls7q7e1sXpbO4s27eLJJQ192+RlpTOuMhL446sKGF+Zz/jKAioKsnVTN0YKeBFJuMz0NMZVFjCusgD6Xe237+9kzfYOVkeDf832Dv64cjuPLNzct03RkEzGV+YzrrKA8RWR0B9XWUBZfpaC/xAKeBFJGgU5mZw1cihnjRz6tvUtHQdYs72dNdvaWb29g7Xb23lqSQNt+7v6tinOzWRcRT5jKwoYV5HPuMpIc1FVYU7KBr8CXkSSXll+NmX52W/rvunuNLUfYO32DtY2tbNmewfrmtp5ZlkjD0XH4gHIz85gTHkeY6L3B8aWR77WluSGfiA2BbyIDEpmRmVhDpWFOW9r33d3WjoOsq6pg3XNHazb3s665g5eWdfCY2/8pUdPZrpRV5rHmPJ8xlREvkYmZMmjMCTDLivgRSRUzIzygmzKC7L7hlnu1ba/kw3NeyLh39TB+uYO1jS188eV2/s+uAWR7pxjyvMigV/WG/55VBcPGVRX/Qp4EUkZhTmZ/6f/PkQGY9u0cy/rmzvY0Lwn+rWDp99s7Bt6GSJX/SNL8xhVlsfossjXUWV5jCrPozw/+Xr3xDXgzexK4L+BdOA+d78nnscTETkRWRlpfX34D7Vzz0E29AZ/Swf1LXvY0LyHF1c39/Xlh0hbf11ZLnWlkfCv6/cHoDg3mJE54xbwZpYOfB94J7AFWGBms9x9RbyOKSIy0ErysijJK2FaXcnb1nf3OA2797GhZQ9vNXdQv2MvG1r2sGTLbp5+s5F+LT4UDcmMBH5pLiNL8/r+ENSV5lGcmxm3K/94XsFPB9a5+wYAM3sYuA5QwIvIoJeeZtSU5FJTkts32UqvA13dbN65j/qWPdTv2MNb0a8L6nfxxJIG+k+kV5iTwYSqAv73b88d8KCPZ8BXA5v7fb8FOOfQjczsDuAOgNra2jiWIyKSGNkZ6Uds8omE/1427thL/Y69bNyxh4NdPXG5ig/8Jqu7zwRmQmRO1oDLERGJq0j4FzC2oiDux4pnf5+tQE2/70dE14mISALEM+AXAOPMbJSZZQEfAmbF8XgiItJP3Jpo3L3LzO4CniXSTfKn7r48XscTEZG3i2sbvLs/DTwdz2OIiMjhDZ7P3IqIyHFRwIuIhJQCXkQkpBTwIiIhZe7J89kiM2sGNh7HS8qAljiVk8x03qlF551ajve8R7p7+eGeSKqAP15mttDdpwVdR6LpvFOLzju1DOR5q4lGRCSkFPAiIiE12AN+ZtAFBETnnVp03qllwM57ULfBi4jIkQ32K3gRETkCBbyISEglfcCb2ZVmttrM1pnZlw7zfLaZPRJ9fp6Z1SW+yoEXw3l/xsxWmNlSM/uzmY0Mos54ONa599vufWbmZhaKrnSxnLeZ3RD9d19uZg8musZ4iOF3vdbMnjezRdHf9z7gwBcAAAY6SURBVKuCqHMgmdlPzazJzJYd4Xkzs+9FfyZLzWzqCR3I3ZN2ITLM8HpgNJAFLAFOPWSbvwN+FH38IeCRoOtO0HlfAuRGH98ZhvOO9dyj2xUALwFzgWlB152gf/NxwCJgaPT7iqDrTtB5zwTujD4+FagPuu4BOO93AFOBZUd4/irgGcCAGcC8EzlOsl/B903c7e4Hgd6Ju/u7Dvh59PFvgMssXlOUJ84xz9vdn3f3vdFv5xKZMSsMYvk3B/gX4D+A/YksLo5iOe+PAd93910A7t6U4BrjIZbzdqAw+rgIaEhgfXHh7i8BO4+yyXXALzxiLlBsZsOO9zjJHvCHm7i7+kjbuHsX0AqUJqS6+InlvPu7nchf+zA45rlH367WuPvvE1lYnMXybz4eGG9mr5jZXDO7MmHVxU8s5/1PwC1mtoXI/BKfTExpgTreDDiswCfdlpNjZrcA04CLgq4lEcwsDfgOcFvApQQhg0gzzcVE3rG9ZGanu/vuQKuKvxuBB9z922Z2LvBLM5vk7j1BF5bskv0KPpaJu/u2MbMMIm/hdiSkuviJacJyM7sc+AfgWnc/kKDa4u1Y514ATAJeMLN6Iu2Ts0JwozWWf/MtwCx373T3t4A1RAJ/MIvlvG8H/hfA3V8DcogMyBVmMWXAsSR7wMcycfcs4CPRx+8HnvPoXYpB7JjnbWZTgB8TCfcwtMX2Ouq5u3uru5e5e5271xG5/3Ctuy8MptwBE8vv+uNErt4xszIiTTYbEllkHMRy3puAywDMbCKRgG9OaJWJNwv4cLQ3zQyg1d0bj3cnSd1E40eYuNvM/hlY6O6zgPuJvGVbR+SmxYeCq3hgxHje3wLygUej95Q3ufu1gRU9QGI899CJ8byfBa4wsxVAN/B5dx/U71ZjPO/PAj8xs78ncsP1tsF+EWdmDxH5Y10WvbfwNSATwN1/RORew1XAOmAv8NcndJxB/nMSEZEjSPYmGhEROUEKeBGRkFLAi4iElAJeRCSkFPAiIiGlgBcRCSkFvIhISCngZVAws24zW2xmy8zsUTPLPYl9/ZOZfS76+NWjbFdsZn93yLojbn8CdQwxsxfNLP0o22SZ2UvRYThEjosCXgaLfe5+prtPAg4CH+//ZPQj3cf9++zu5x3l6WIi8w3Euv3x+ijwmLt3H2mD6BC6fwY+OIDHlRShgJfBaA4w1szqojMB/QJYBtSY2S1mNj96tf/j3qtjM/sHM1tjZi8DE3p3ZGYd/R5/ODp7zhIz+yVwDzAmuq9vHWb7z0TfUSwzs09H19WZ2Uoz+0l01qXZZjbkCOdxM/BEv/09ZJHZyeab2UYze0/0qcej24ocFwW8DCrRpop3A29GV40DfuDupwG5RK50z3f3M4mM13KzmZ1FZIyiM4mM73H2YfZ7GvBV4FJ3nwzcDXwJWB995/D5Q7Y/i8j4IOcQGdHyY9EB4Hpr+n60pt3A+w5zvCxgtLvX91s9Gdjg7tOJBPrXouuXHa5mkWNRu54MFkPMbHH08Rwig8wNBzZGZ7yByIiDZwELogOwDQGagBLgd70zYJnZ4QYsuxR41N1bANx9p5kVHma7XhdE97knus/HgAuJjAL4lrv31vo6UHeY15cRCX+ir88ByoGvR1etAIZGa+k2s4NmVuDu7UepSeRtFPAyWOyLXpX3iYb4nv6rgJ+7+5cP2e7T8S/vbfqPzd9N5A/NofYRGfa21yRgrbv3TkE4lcj8pL2yCc/0hJIgaqKRMPkz8H4zqwAwsxIzG0lkcu7ro71WCoBrDvPa54APmFlp72uBdiITjBzOnOg+c80sD3hvdF1MovOqpkev3CHSPFNrZjnR/X0d+K9oLaVAi7t3xrp/EVDAS4i4+woi7eizzWwp8EdgmLu/ATxC5Ir4GSKTTBz62uXAvwIvmtkS4DvRsdZfid5E/dYh278BPADMB+YB97n7ouMseTaRph6IBPxj0X0tAH7o7q9En7sECNP8s5IgGg9eJCAWmTz87939VjN7EbjD3VcfZrvHgC+5+5qEFymDmq7gRQISfRfwfLQr5xhg7aHbRHvbPK5wlxOhK3gRkZDSFbyISEgp4EVEQkoBLyISUgp4EZGQUsCLiISUAl5EJKQU8CIiIfX/Aa0FoUc/4UnsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.linspace(0.01, 0.99, 99)\n",
    "y = -1.0 * np.log((x))\n",
    "\n",
    "plt.plot(x, y);\n",
    "plt.title(\"Log loss for $y = 1$\")\n",
    "plt.xlabel(\"Prediction ($p$)\")\n",
    "plt.ylabel(\"Log loss\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Derivation of Softmax Cross Entropy derivative (simple case, two probabilities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ C(p, y) = - y * \\text{log}(p) - (1 - y) * \\text{log}(1-p) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\n",
    "C(p,y)=\n",
    "\\begin{cases}\n",
    "-log(1-p) & \\text{if }  y = 0\\\\\n",
    "-log(p) & \\text{if }  y = 1\n",
    "\\end{cases}\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ \\text{SCE}(\\begin{bmatrix} p_1 \\\\ p_2 \\end{bmatrix}, \\begin{bmatrix} y_1 \\\\ y_2 \\end{bmatrix})_1 = - y_1 * log(S(\\begin{bmatrix} p_1 \\\\ p_2 \\end{bmatrix})_1) - (1 - y_1) * log(1-S(\\begin{bmatrix} p_1 \\\\ p_2 \\end{bmatrix})_1) = - y_1 * log(\\frac{e^{p_1}}{e^{p_1} + e^{p_2}}) - (1 - y_1) * log(1-\\frac{e^{p_1}}{e^{p_1} + e^{p_2}}) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Structure:\n",
    "\n",
    "$ y_1 = a $, $e^{x_2} = b$\n",
    "\n",
    "$$ SC(x_1) = - a * log(\\frac{e^{x_1}}{e^{x_1} + b}) - (1 - a) * log(1-\\frac{e^{x_1}}{e^{x_1} + b}) $$\n",
    "\n",
    "**Quotient rule:**\n",
    "\n",
    "$$ f(x) = \\frac{g(x)}{h(x)} $$ \n",
    "\n",
    "$$ f'(x) = \\frac{g'(x) * h(x) - g(x) * h'(x)}{(h(x))^2} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If\n",
    "\n",
    "$$ f(x) = \\frac{e^x}{e^x + b} $$ \n",
    "\n",
    "$$ \\begin{align} f'(x) =& \\frac{e^x * (e^x + b) - (e^x * e^x)}{(e^x + b)^2} \\\\ \n",
    "=& \\frac{e^x * (e^x + b - e^ x)}{(e^x + b)^2} \\\\\n",
    "=& \\frac{e^x * b}{(e^x + b)^2}\\end{align} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And if:\n",
    "\n",
    "$$ g(x) = - a * log(f(x)) - (1 - a) * log(1-f(x)) $$\n",
    "\n",
    "then:\n",
    "\n",
    "$$ g'(x) = - a * \\frac{f'(x)}{f(x)} - (1 - a) * \\frac{-1 * f'(x)}{1-f(x)} $$\n",
    "\n",
    "First, we'll compute $\\frac{f'(x)}{f(x)}$:\n",
    "\n",
    "$$ \\begin{align} \n",
    "\\frac{f'(x)}{f(x)} =& \\frac{\\frac{e^x * b}{(e^x + b)^2}}{\\frac{e^x}{e^x + b}} \\\\\\\\\n",
    "=& \\frac{-e^x * b}{(e^x + b)^2} * \\frac{e^x + b}{e^x}\n",
    " \\\\\n",
    "=& \\frac{b}{e^x + b} \\end{align} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, in this next part, we'll use the fact that:\n",
    "\n",
    "$$ \\frac{b}{e^x + b} = 1 - \\frac{e^x}{e^x + b} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$ \\begin{align} \n",
    "\\frac{-1 * f'(x)}{1 - f(x)} =& \\frac{-1 * \\frac{e^x * b}{(e^x + b)^2}}{1 - \\frac{e^x}{e^x + b}} \\\\\n",
    "=& \\frac{\\frac{-e^x * b}{(e^x + b)^2}}{\\frac{b}{e^x + b}} \\\\\n",
    "=& \\frac{-e^x * b}{(e^x + b)^2} * \\frac{e^x + b}{b}\n",
    " \\\\\n",
    "=& \\frac{-e^x}{e^x + b} \\end{align}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, putting these pieces together:\n",
    "\n",
    "$$ \\begin{align} SC'(x) =& - a * \\frac{f'(x)}{f(x)} - (1 - a) * \\frac{-1 * f'(x)}{1-f(x)} \\\\ \n",
    "=& -a * \\frac{b}{e^x + b} - (1 - a) * \\frac{-e^x}{e^x + b} \\\\ \n",
    "=& -a * \\frac{b}{e^x + b} + \\frac{e^x}{e^x + b} - a * \\frac{-e^x}{e^x + b} \\\\ \n",
    "=& -a * (1 - \\frac{e^x}{e^x + b}) + \\frac{e^x}{e^x + b} - a * \\frac{-e^x}{e^x + b} \\\\\n",
    "=& -a + a * \\frac{e^x}{e^x + b} + \\frac{e^x}{e^x + b} - a * \\frac{-e^x}{e^x + b} \\\\\n",
    "=& -a + \\frac{e^x}{e^x + b} \\\\\n",
    "\\end{align} \\\\ \n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's right, the derivative to be sent backward from the softmax layer is simply:\n",
    "\n",
    "$$ S - Y = s(\\begin{bmatrix} p_1 \\\\ p_2 \\end{bmatrix}) - \\begin{bmatrix} y_1 \\\\ y_2 \\end{bmatrix} = \\begin{bmatrix} \\frac{e^{p_1}}{e^{p_1} + e^{p_2}} - y_1 \\\\ \\frac{e^{p_2}}{e^{p_1} + e^{p_2}} - y_2 \\end{bmatrix} $$\n",
    "\n",
    "This makes sense:\n",
    "\n",
    "* The softmax output will always be between 0 and 1.\n",
    "* If $y_i$ is 0, then $ s(x_1) - y_1 $ will be a positive number: because indeed, if we increase the value of $x_1$, the loss will increase. Conversely if $y_i$ is one.\n",
    "* Note that this loss function only makes sense because $ s(x_i) $ is always between 0 and 1. \n",
    "\n",
    "This, by the way, is why TensorFlow has a function called `softmax_cross_entropy_with_logits`!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numerically stable softmax math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ log(softmax(x_j)) = log(\\frac{e^{x_j}}{\\sum_i^n e^{x_j}}) = x_j - logsumexp(X) $$\n",
    "\n",
    "$$ softmax(x_j) = e^{log(softmax(x_j))} = e^{x - logsumexp(x)} $$\n",
    "\n",
    "$$ logsumexp(X) = log(\\sum_i^n e^{x_i}) = c + log(\\sum_i^n e^{x_i - c}) $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# One hot encoding\n",
    "\n",
    "$$ [0, 2, 1] \\Rightarrow \\begin{bmatrix} 1 & 0 & 0 & \\ldots & 0 \\\\ 0 & 0 & 1 & \\ldots & 0 \\\\ 0 & 1 & 0 & \\ldots & 0 \\end{bmatrix} $$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning rate decay"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear decay\n",
    "\n",
    "$$ \\alpha_{t} = \\alpha_{start} - (\\alpha_{start} - \\alpha_{end}) * \\frac{t}{N} $$\n",
    "\n",
    "## Exponential decay\n",
    "\n",
    "$$ \\alpha_{t} = \\alpha_{start} * ((\\frac{\\alpha_{end}}{\\alpha_{start}})^\\frac{1}{N})^t $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SGD Momentum gradient formula"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$ \\text{update} = \\nabla_t + \\mu * \\nabla_{t-1} + \\mu^2 * \\nabla_{t-2} + \\ldots $"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weight init illustrations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature level math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$ f_n = w_{1, n}* x_1 + \\ldots + w_{784, n}* x_{784} + b_n $"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$ \\text{Var}(w_{i,j}) = 1 $"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$ \\text{Var}(X_1 + X_2) = \\text{Var}(X_1) + \\text{Var}(X_2) $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_feat = 784\n",
    "n_hidden = 256\n",
    "\n",
    "np.random.seed(190131)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.random.randn(1, n_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = np.random.randn(n_feat, n_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = np.dot(a, b).reshape(n_hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAFOCAYAAACPP3XPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dfZxtdV33/9dbjqB4w40cCbk7mKih/UQ6oV7eBmYoJlRehJmh4kWWmaVlkHWllYWXXd6VV0aAgiKCqIFiJSJIlqCogAjeHBDkHG7OAQFRE0E/vz/Wd2SfYebMnpk9s2dmvZ6Pxzxm3a/P/u61v/uz1/qu9U1VIUmSJPXBfcYdgCRJkrRYTH4lSZLUGya/kiRJ6g2TX0mSJPWGya8kSZJ6w+RXkiRJvdGr5DfJu5L8+Yi2tUeS7ybZqo2fn+Rlo9h2296/JjliVNubxX7/OsnNSW6cYt5Tk3xtsWNarpK8MMknFmjbIzuWJ203Sd6d5NYknxv19mfY91iOea0s1vND7Xfaer4vkvxKkuva+/v4Bd7XmiSVZNWQy98/yUeT3J7kgwsZ2xT7/kqSZyzmPschK+U5v0muAXYG7gZ+BFwBnAwcV1U/nsO2XlZVn5zFOucD76uq42ezr7bu64FHVNVvznbdUUqyB/A1YM+q2jjmWArYu6rWLea6c5VkDfBN4L5VdfeIt/1iuuPxKaPc7jT7eipwKvCoqvreAu7n9SyBY17Li/X8/C1kPT/qumoh674kVwGvrqozp5k/su+R2X4/JHkR8Ergf4z6+2TSft4DrK+qP1uofSxVK+3M7y9X1YOAPYFjgT8BThj1Tob99bYM7QHcMu7EV2O1J3DNQia+0jxZz8+P9XxnT+Ar4w5iGnsCX1/IxLf3qmpF/AHXAM+cNG1/4MfAY9v4e4C/bsM7AR8DbgO+DfwH3Y+B97Z1/hv4LvBaYA1QwJHAt4ALBqatats7H/hb4HPAd4AzgR3bvGfQ/bq6V7zAQcAPgbva/i4d2N7L2vB9gD8DrgU20p3p2K7Nm4jjiBbbzcDrtlBO27X1N7Xt/Vnb/jPba/5xi+M9U6y72etor+GPgMuA24HTgPsNLgv8aYvpGuCFA+v+5PW18RcDn2nDF7TX9L0Wy69P935NEeO91m3T/xewrq17FvCwLZTRB4Eb22u6AHjMwLz7A/+3ld3twGfatG+1/X63/T1p0mv6R+DvJu3nTLozDwBHA1cBd9CdzfqVNv1ngB/QneX6LnDb5GN5ptfX4no58I1Wfu+kXfWZFM+Rk/b1hsHXMGl7jxiI453A2S32i4CfHlj2McA5La6b2vGwqMe8fyvnD+v5Ba/n2/pT1ieTy2PwNbDluupddPXAHcCn6c46z3V7z6GrI+8ANgB/NM1rmLI8gW3a9ia+J66aYt2pvoN2oDuWNgG3tuHdJsX9V8B/ttg+Aew02/ePrt4dPFaOBF5Pd8WBqcptS/tu858C/Bfd5+A6unr9qLaPH7b9fHTyZ6yV1duA69vf24BtBo934DWtfG8AXjLuOmLoumTcAYzshUxRKbbp3wJ+Z+BDOFEp/i3dB/K+7e+p3NMMZLNtDRxoJwMPoEt2pjr4NgCPbct8aOJgZQuVYhve7MAe2N5EpfhSuoro4cADgQ8D750U2z+3uB4H3An8zDTldDJdhf2gtu7XgSOni3PSupvNb6/hc8DDgB2BK4GXDyx7N/CW9gF6Ol1F8qjJr6+Nv5iBJIuBBGum92uKOCevewBdZbNfi+XvgQu28Dpf2spn4oN/ycC8d7bYdwW2Av5HW26z42HyawKeRlfpTBxjO9B9CU18qfzPVo73oatovwfsMlXZTHEsb/H1tbg+BmxPd9ZnE3DQNK998vsw1b4nJ7+30CUgq4BTgA+0eQ+iqxBfA9yvjT9hsY95/1bOH9bzi1HPT1ufTC6PKV7Di5m6rrqDrg7cBng799SLc9neDcBT2/AOwH7TvI5py7PN3+x7Yor1J3+PPAT4NWDbVq4fBP5lUtxXAY9s79H5wLFzfP82O1amGN+s3GbY956t/F9A9xl4CLDv5M/KNMfsXwIXAg8FVtMl0H81cBzd3Za5L92Pku8DO4y7nhjmb6U1e5jK9XSJ2WR3AbvQ/QK9q6r+o9o7ugWvr6rvVdV/TzP/vVV1eXWXjP8cOGziRol5eiHwlqq6uqq+CxwDHD7pstwbquq/q+pS4FK6D9dmWiyHA8dU1R1VdQ3dWcwXzSO2d1TV9VX1beCjwL6T5v95Vd1ZVZ+mOzt42Bz3M5f3a8ILgROr6otVdSdd+T2ptcO6l6o6sZXPnXSVzuOSbJfkPnQV6quqakNV/aiq/qstN5P/oKusntrGnw98tqqub/v8YCvHH1fVaXRnafcf4es7tqpuq6pvAedx7/dpPj5SVZ+r7hLdKQPbfi5wY1X936r6QSvTi4bc5kiOefWG9Xwzgnp+VvXlkM6uqgva9l7Xtrf7HLd1F7BPkgdX1a1V9cVplhumPIdWVbdU1Yeq6vtVdQfwRrqTOoPeXVVfb8fO6dy7nl3IOmu6ff8G8MmqOrV9Bm6pqkuG3OYLgb+sqo1VtYnurPTgcXRXm39XVX2c7gzyo0bzchZWH5LfXeku3Uz2ZrpfhZ9IcnWSo4fY1nWzmH8t3a+hnYaKcsse1rY3uO1VdDd+TBi8a/f7dL90J9upxTR5W7vOI7Yt7ffW2rzt6LV0r2Uu5vJ+Tdis/FpFeAtTvO4kWyU5NslVSb5D9ysYurLbie4M5lWzDb594X6A7tc3dBXSKQP7/a0klyS5LcltdGeWhj12hnl9wxwfczXdtndnDmXVjOqYVz9Yz99jvvX80PXlLPykzNr2vs3cvwt+je4s47VJPp3kSdMsN0x5Di3Jtkn+Kcm17bvhAmD7ST98Znp/VkI9PPi+3VKbt0teNvXwik5+k/w83Qf2M5PntV/Er6mqhwPPA16d5MCJ2dNscqYzBoO/ZPeg+1V0M90l7G0H4tqK7hLCsNu9nu7SxeC276ZrQzkbN7eYJm9rwyy3M6wdkjxg0r6ub8OblQnwU1va0Azv10w2K78W00OY+nX/BnAIXdu47eguLwGErvx+APz0VCEOEcepwPOT7Ak8ge6SKW38n4HfAx5SVdsDl7d9DrPt2by+2Zp87G7xfZrkOrpLjlNZrGNeK5z1/L3Mt57fUn0ycTJjurp7utf4kzJL8kC6s/TXz2V7VfX5qjqE7lL8v9Cd5ZzxdTD/OuQ1dGc1n1BVD6ZrxgH31NMLaVbfl5Ncx9TfWTC3Y/L6aZZdVlZk8pvkwUmeS3em7X1V9eUplnlukkckCd2NSz+iuwkAug/HdF/aW/KbSfZJsi1dO5gzqupHdO2t7pfk4CT3pWuEv83AejcBa9pl9amcCvxhkr1axfE3wGk1yztBWyynA29M8qCWdL0aeN9stjNLb0iydXuE1nPp2kkBXAL8avs1/Qi6Rv2DNnsPZni/Jpv8/p0KvCTJvkm2oSu/i9rlwMkeRNcW6xa6yuZvJmZU9yilE4G3JHlYO0v8pLbNTS2eaY+bqvoS3RfT8cC/V9VtbdYD6CqhTe21voTuzO/g69ktydbTbHo2r2+2LgUe07Z9P7pmIMP6GLBLkj9Isk075p7Q5i3KMa+Vy3p+aiOo56etT9ql7w2tDLZK8lI2T6ymq6uek+QpbfpfARdW1XWz3V77Lnlhku2q6i66mw6n+x6Yb3lOPj4eRHefxm1JdgT+YsjtjMIlwNPSPXd6O7omHMM6BXhmksOSrErykCQTTSJm+gycCvxZktVJdgL+NwubLyyalZb8fjTJHXS/dF5Hd7PVS6ZZdm/gk3RtVD4L/L+qOq/N+1u6N/y2JH80i/2/l64B+Y10l8d/H6Cqbgd+ly7pmfj1vH5gvYmE8JYkU7VfOrFt+wK6ZwX+gO4ZgHPxyrb/q+nOlLy/bX8h3Eh3V+z1dB/Al1fVV9u8t9LdZXoTcBIDTQCa1wMntffgMLb8fk222brVPcfzz+nOtN5AV7kePs26J9Nd2tlAd0fxhZPm/xHwZeDzdJfu3kT31Inv07UB+8+23ydOs/33051Vfv/EhKq6gq5N3mdbefws3V27Ez5F90ieG5PcPHmDs3x9s1JVX6f7gv8kXTvke51d28K6dwC/CPwy3bHwDeAX2uzFPOa1sljPz2zO9fwQ9cn/Av6Y7gTBY+hugpowXV31frpk8dvAzwGDzzqe7fZeBFzTmh68nK5d6lTmW56vZ/PvoLfR3Ux2M933wr/NYlvzUlXn0D1N6TLgC3QnFoZd91t0zUReQ1f+l3BPW+MT6NpP35bkX6ZY/a+Bi9t+vwx8sU1b9lZMJxdaWtL1EPO+qtpt3LFIksYjPe5IQUvXSjvzK0mSJE3L5FeSJEm9YbMHSZIk9YZnfiVJktQbJr/LRJJHpesE4Y4kvz+C7b0+ybJ5ZEl7vM0ntjD/GUnWTzd/nJJ8N8lcHqkkaZmyzrbO1tJl8rt8vBY4r6oeVFXvGHcwi62qTqmqZ02MJ6n2fOA5SfKVVsFN/N2d5KMD8w9I8sUk30nXM9RR02znxJliqaoHVtXVc411WEnOT/Kyhd6PpKFYZ1tnb5F19viY/C4fe9I973DWMse+zEdtqcQBUFWPaRXcA+keXn4d7Tmc7QH1HwH+ia6Xt1+n69his37YkzyF6XvOkdRv1tkjZJ2tUTL5XQaSfIquc4B/aL94H5lkuyQnJ9mUrq/xP0vrOSjJi5P8Z5K3JrmFIXrlSvLBJDcmuT3JBUke06b/fJKbMtB/eZJfTXJpG75PkqOTXJXkliSnp+v9hiRr2i/sI5N8i+6B5ZP3++kkv9aGn9yWP7iNH5jkkoHX9Jk2fEFb/dJWHr8+sL3XJNmY5IZ0PaUN42nATrTuhum63nww8N7qfB64EthnYD+rgL9niIemD55lSPKeJO9Mcna7HHpRkp+etOzvtzMXNyd588D7utllz4HyXZXkjcBTuecY+Yd03trK4ztJvpzksZPjkzRa1tnW2W2edfYSZfK7DFTVAcB/AL/Xfvl+ne5DvB1d14RPB36LzXs5egJd7z470/U8NpN/pesN6aF0vbic0vb9ebqed541sOyL6HpCg64iObTF8DC6Ht3eOWnbTwd+BvilKfb7aeAZA8tdzT19pj+9zd9MVU3Mf1wrj9Pa+E/RlcmudN0lvzPJDlt4zROOAD5UVd9r27+Je7r43CrJk+jO4gz2bvaHwAVVddkQ25/scOANwA7AOu79/vwKsBbYDzgEeOlMG6yq17H5MfJ7dO/Z04BH0pXLYXTvpaQFZJ29Oevse7POHi+T32Wo/aI/HDimqu6oqmvousd90cBi11fV31fV3VX13zNts6pObNu6k+6sw+PS9SEOXffDv9n2vSNdhTjRPe/LgddV1fqBdZ+fzS+Xvb6qvjdNHJ+mqzCh+9D/7cD4lBXpFtwF/GVV3VVVH6fr0vRRW1ohybbA8+m6Kx10Kl0/5nfSVVCvq6rr2jq7A7/d5s/FR6rqc62P+VOAfSfNf1NVfbt1S/k24AVz3M9ddJcHH033WMMrq+qGOW5L0hxZZ0/LOntz1tmLxOR3edoJuC9w7cC0a+l+PU+4btiNtV/Kx7bLYN8BrhnYD8D7gF9O8gC6X6L/MfCB3BP4SLq+wW+ju9T0I7qzF8PE8lngkUl2pqtQTgZ2T7ITsD9dv+zDuqVVThO+DzxwhnV+la6/859U2EkeDXyA7szM1nT9zb924tIeXeX2l1V1+yxiG3TjDDEOlte1dGdnZq2qPgX8A91ZnY1Jjkvy4LlsS9K8WGdPzTp7gHX24jH5XZ5upvuFuOfAtD2ADQPjs+m95DfoLtU8k+5Sy5o2PQBVtYGuwvtVujMV7x1Y9zrg2VW1/cDf/do6M8ZSVd8HvgC8Cri8qn4I/BfwauCqqrp5Fq9jLo4ATq7Ne3t5LPD1qvr3qvpxVX0NOBt4dpt/IPDm1t5uolL8bJLfGFFMuw8M7wFc34a/B2w7MO+nJq13r3KuqndU1c/RtX17JPDHI4pR0vCss0fHOlvzZvK7DFXVj4DTgTcmeVCSPekqnrk+A/JBdJeKbqH7oP7NFMucTPfonp8FPjww/V0tjj0BkqxOcsgs9/9p4Pe455f8+ZPGp3ITXdu5OUuyG91NKSdNmvUlYO90j85Ju7nhucBEW7FHAo+jO+sxcfnrl+nuNh6FP06yQ7tU9ypgon3cJcDTkuzRLm8eM2m9zcok3Y0vT0h3J/T3gB8APx5RjJKGZJ0NWGdbZy8hJr/L1yvpPhxX0zXqfz9w4hy3dTLdpZoNwBXAhVMs8xHa5bL2y3/C24GzgE8kuaOt+4RZ7v/TdJX5BdOMT+X1wEnt0t1hs9zfhBcBn62qqwYntvGXAu8AvtPi+RBwfJu/sapunPhrq908TDu9IZ1Jd2blErqzFye0/Z5DV6le1uZ/bNJ6b6dru3drknfQ3f38z3Q3tFxL90X55hHFKGl2rLOts62zl4hsfuVAml6Sq4DfrqpPjjuWlSpJAXtX1bpxxyJpebPOXnjW2cuTZ341lHTPdSymeO6jJGlpsc6Wprdkem/R0pXkfLrG9y+qKtsfSdISZp0tbZnNHiRJktQbNnuQJElSb5j8SpIkqTcWtc3vTjvtVGvWrFnMXUrSSHzhC1+4uapWjzuOxWSdLWm52lKdvajJ75o1a7j44osXc5eSNBJJrp15qZXFOlvScrWlOttmD5K0DCU5McnGJJdPMe81SSrJTm08Sd6RZF2Sy5Lst/gRS9LSYPIrScvTe4CDJk9sXaw+C/jWwORnA3u3v6OAf1yE+CRpSTL5laRlqKouAL49xay3Aq+l6+BgwiHAydW5ENg+yS6LEKYkLTkmv5K0QiQ5BNhQVZdOmrUrcN3A+Po2baptHJXk4iQXb9q0aYEilaTxMfmVpBUgybbAnwL/ez7bqarjqmptVa1dvbpXD7eQ1BN2byxJK8NPA3sBlyYB2A34YpL9gQ3A7gPL7tamSVLveOZXklaAqvpyVT20qtZU1Rq6pg37VdWNwFnAb7WnPjwRuL2qbhhnvJI0Lia/krQMJTkV+CzwqCTrkxy5hcU/DlwNrAP+GfjdRQhRkpYkmz1I0jJUVS+YYf6ageECXrHQMUnScuCZX0mSJPWGya8kSZJ6Y6hmD0m2B44HHkv34PSXAl8DTgPWANcAh1XVrQsSpTSENUefvaj7u+bYgxd1f5IkLQXL/ft22DO/bwf+raoeDTwOuBI4Gji3qvYGzm3jkiRJ0pI1Y/KbZDvgacAJAFX1w6q6ja67zJPaYicBhy5UkJIkSdIoDHPmdy9gE/DuJF9KcnySBwA7Dzwn8kZg54UKUpIkSRqFYZLfVcB+wD9W1eOB7zGpiUN7jE5NtbL9xEuSJGmpGCb5XQ+sr6qL2vgZdMnwTUl2AWj/N061sv3ES5IkaamYMfltXWNel+RRbdKBwBV03WUe0aYdAZy5IBFKkiRJIzJsD2+vBE5JsjVdF5kvoUucT29dal4LHLYwIUqSJEmjMVTyW1WXAGunmHXgaMORJEmSFo49vEmSJKk3TH4lSZLUGya/kiRJ6g2TX0mSJPWGya8kSZJ6w+RXkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN0x+JUmS1Bsmv5IkSeoNk19JkiT1hsmvJEmSesPkV5IkSb1h8itJkqTeMPmVJElSb5j8SpIkqTdMfiVJktQbJr+StAwlOTHJxiSXD0x7c5KvJrksyUeSbD8w75gk65J8LckvjSdqSRo/k19JWp7eAxw0ado5wGOr6v8Dvg4cA5BkH+Bw4DFtnf+XZKvFC1WSlg6TX0lahqrqAuDbk6Z9oqrubqMXAru14UOAD1TVnVX1TWAdsP+iBStJS4jJryStTC8F/rUN7wpcNzBvfZt2L0mOSnJxkos3bdq0wCFK0uIz+ZWkFSbJ64C7gVNmu25VHVdVa6tq7erVq0cfnCSN2apxByBJGp0kLwaeCxxYVdUmbwB2H1hstzZNknrHM7+StEIkOQh4LfC8qvr+wKyzgMOTbJNkL2Bv4HPjiFGSxs0zv5K0DCU5FXgGsFOS9cBf0D3dYRvgnCQAF1bVy6vqK0lOB66gaw7xiqr60Xgil6TxMvmVpGWoql4wxeQTtrD8G4E3LlxEkrQ82OxBkiRJvWHyK0mSpN6w2YMkaaTWHH32ou7vmmMPXtT9SVrePPMrSZKk3jD5lSRJUm+Y/EqSJKk3TH4lSZLUG0Pd8JbkGuAO4EfA3VW1NsmOwGnAGuAa4LCqunVhwpQkSZLmbzZnfn+hqvatqrVt/Gjg3KraGzi3jUuSJElL1nyaPRwCnNSGTwIOnX84kiRJ0sIZNvkt4BNJvpDkqDZt56q6oQ3fCOw88ugkSZKkERq2k4unVNWGJA8Fzkny1cGZVVVJaqoVW7J8FMAee+wxr2AlSZKk+RjqzG9VbWj/NwIfAfYHbkqyC0D7v3GadY+rqrVVtXb16tWjiVqSJEmagxmT3yQPSPKgiWHgWcDlwFnAEW2xI4AzFypISZIkaRSGafawM/CRJBPLv7+q/i3J54HTkxwJXAsctnBhSpIkSfM3Y/JbVVcDj5ti+i3AgQsRlCRJkrQQ7OFNkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN0x+JUmS1Bsmv5IkSeoNk19JkiT1hsmvJEmSesPkV5IkSb1h8itJkqTeMPmVJElSb5j8SpIkqTdMfiVJktQbJr+StAwlOTHJxiSXD0zbMck5Sb7R/u/QpifJO5KsS3JZkv3GF7kkjZfJryQtT+8BDpo07Wjg3KraGzi3jQM8G9i7/R0F/OMixShJS47JryQtQ1V1AfDtSZMPAU5qwycBhw5MP7k6FwLbJ9llcSKVpKXF5FeSVo6dq+qGNnwjsHMb3hW4bmC59W3avSQ5KsnFSS7etGnTwkUqSWNi8itJK1BVFVBzWO+4qlpbVWtXr169AJFJ0niZ/ErSynHTRHOG9n9jm74B2H1gud3aNEnqHZNfSVo5zgKOaMNHAGcOTP+t9tSHJwK3DzSPkKReWTXuACRJs5fkVOAZwE5J1gN/ARwLnJ7kSOBa4LC2+MeB5wDrgO8DL1n0gCVpiTD5laRlqKpeMM2sA6dYtoBXLGxEkrQ82OxBkiRJvWHyK0mSpN6w2UPPrDn67EXb1zXHHrxo+5IkSRqGZ34lSZLUGya/kiRJ6g2TX0mSJPWGya8kSZJ6wxvetGAW8+Y6SZKkYXjmV5IkSb1h8itJkqTeMPmVJElSbwyd/CbZKsmXknysje+V5KIk65KclmTrhQtTkiRJmr/ZnPl9FXDlwPibgLdW1SOAW4EjRxmYJEmSNGpDJb9JdgMOBo5v4wEOAM5oi5wEHLoQAUqSJEmjMuyZ37cBrwV+3MYfAtxWVXe38fXAriOOTZIkSRqpGZ/zm+S5wMaq+kKSZ8x2B0mOAo4C2GOPPWYdoLRULeZzjK859uBF25ckSSvZMGd+nww8L8k1wAfomju8Hdg+yUTyvBuwYaqVq+q4qlpbVWtXr149gpAlSZKkuZkx+a2qY6pqt6paAxwOfKqqXgicBzy/LXYEcOaCRSlJkiSNwHye8/snwKuTrKNrA3zCaEKSJEmSFsaMbX4HVdX5wPlt+Gpg/9GHJEmSJC0Me3iTJElSb5j8SpIkqTdMfiVJktQbJr+SJEnqDZNfSZIk9YbJryRJknrD5FeSJEm9YfIrSZKk3jD5lSRJUm+Y/ErSCpPkD5N8JcnlSU5Ncr8keyW5KMm6JKcl2XrccUrSOJj8StIKkmRX4PeBtVX1WGAr4HDgTcBbq+oRwK3AkeOLUpLGx+RXklaeVcD9k6wCtgVuAA4AzmjzTwIOHVNskjRWJr+StIJU1Qbg74Bv0SW9twNfAG6rqrvbYuuBXccToSSNl8mvJK0gSXYADgH2Ah4GPAA4aBbrH5Xk4iQXb9q0aYGilKTxMfmVpJXlmcA3q2pTVd0FfBh4MrB9awYBsBuwYaqVq+q4qlpbVWtXr169OBFL0iIy+ZWkleVbwBOTbJskwIHAFcB5wPPbMkcAZ44pPkkaK5NfSVpBquoiuhvbvgh8ma6ePw74E+DVSdYBDwFOGFuQkjRGq2ZeRJK0nFTVXwB/MWny1cD+YwhHkpYUz/xKkiSpN0x+JUmS1Bsmv5IkSeoNk19JkiT1hsmvJEmSesPkV5IkSb1h8itJkqTeMPmVJElSb5j8SpIkqTdMfiVJktQbJr+SJEnqDZNfSZIk9YbJryRJknrD5FeSJEm9YfIrSZKk3jD5lSRJUm/MmPwmuV+SzyW5NMlXkryhTd8ryUVJ1iU5LcnWCx+uJEmSNHfDnPm9Ezigqh4H7AsclOSJwJuAt1bVI4BbgSMXLkxJkiRp/mZMfqvz3TZ63/ZXwAHAGW36ScChCxKhJEmSNCJDtflNslWSS4CNwDnAVcBtVXV3W2Q9sOs06x6V5OIkF2/atGkUMUuSJElzMlTyW1U/qqp9gd2A/YFHD7uDqjquqtZW1drVq1fPMUxJkiRp/mb1tIequg04D3gSsH2SVW3WbsCGEccmSZIkjdQwT3tYnWT7Nnx/4BeBK+mS4Oe3xY4AzlyoICVJkqRRWDXzIuwCnJRkK7pk+fSq+liSK4APJPlr4EvACQsYpyRJkjRvMya/VXUZ8Pgppl9N1/5XkiRJWhbs4U2SJEm9YfIrSZKk3jD5lSRJUm+Y/EqSJKk3TH4laYVJsn2SM5J8NcmVSZ6UZMck5yT5Rvu/w7jjlKRxMPmVpJXn7cC/VdWjgcfRPZv9aODcqtobOLeNS1LvmPxK0gqSZDvgabRnr1fVD1vvnIcAJ7XFTgIOHU+EkjReJr+StLLsBWwC3p3kS0mOT/IAYOequqEtcyOw89gilKQxMvmVpJVlFbAf8I9V9Xjge0xq4lBVBdRUKyc5KsnFSS7etGnTggcrSYvN5FeSVpb1wPqquqiNn0GXDN+UZBeA9n/jVCtX1XFVtbaq1q5evXpRApakxWTyK0krSFXdCFyX5FFt0oHAFcBZwBFt2hHAmWMIT5LGbtW4A5AkjdwrgVOSbA1cDbyE7mTH6UmOBK4FDhtjfJI0Nia/kncp0AgAABDmSURBVLTCVNUlwNopZh242LFI0lJjswdJkiT1hsmvJEmSesPkV5IkSb1h8itJkqTeMPmVJElSb5j8SpIkqTdMfiVJktQbJr+SJEnqDZNfSZIk9YbJryRJknrD5FeSJEm9sWrcAUia2Zqjz17U/V1z7MGLuj9JkhaLZ34lSZLUGya/kiRJ6g2TX0mSJPWGya8kSZJ6w+RXkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN2ZMfpPsnuS8JFck+UqSV7XpOyY5J8k32v8dFj5cSZIkae6GOfN7N/CaqtoHeCLwiiT7AEcD51bV3sC5bVySJElasmZMfqvqhqr6Yhu+A7gS2BU4BDipLXYScOhCBSlJkiSNwqza/CZZAzweuAjYuapuaLNuBHYeaWSSJEnSiK0adsEkDwQ+BPxBVX0nyU/mVVUlqWnWOwo4CmCPPfaYX7Qr0Jqjzx53CJIkSb0x1JnfJPelS3xPqaoPt8k3Jdmlzd8F2DjVulV1XFWtraq1q1evHkXMkiRJ0pwM87SHACcAV1bVWwZmnQUc0YaPAM4cfXiSJEnS6AzT7OHJwIuALye5pE37U+BY4PQkRwLXAoctTIiSJEnSaMyY/FbVZ4BMM/vA0YYjSZIkLRx7eJOkFSjJVkm+lORjbXyvJBclWZfktCRbjztGSRoHk19JWpleRfdc9glvAt5aVY8AbgWOHEtUkjRmJr+StMIk2Q04GDi+jQc4ADijLWLHRJJ6y+RXklaetwGvBX7cxh8C3FZVd7fx9XQ9dUpS75j8StIKkuS5wMaq+sIc1z8qycVJLt60adOIo5Ok8TP5laSV5cnA85JcA3yArrnD24Htk0w84Wc3YMNUK9sxkaSVzuRXklaQqjqmqnarqjXA4cCnquqFwHnA89tidkwkqbdMfiWpH/4EeHWSdXRtgE8YczySNBbD9PAmSVqGqup84Pw2fDWw/zjjkaSlwDO/kiRJ6g2TX0mSJPWGya8kSZJ6w+RXkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN0x+JUmS1Bsmv5IkSeoNk19JkiT1hsmvJEmSesPkV5IkSb1h8itJkqTeMPmVJElSb5j8SpIkqTdMfiVJktQbJr+SJEnqDZNfSZIk9YbJryRJknrD5FeSJEm9YfIrSZKk3jD5lSRJUm+Y/EqSJKk3TH4lSZLUG6vGHYCkpWfN0Wcv6v6uOfbgRd2fJKm/Zjzzm+TEJBuTXD4wbcck5yT5Rvu/w8KGKUmSJM3fMM0e3gMcNGna0cC5VbU3cG4blyRJkpa0GZPfqroA+PakyYcAJ7Xhk4BDRxyXJEmSNHJzveFt56q6oQ3fCOw83YJJjkpycZKLN23aNMfdSZIkSfM376c9VFUBtYX5x1XV2qpau3r16vnuTpK0BUl2T3JekiuSfCXJq9p079WQJOae/N6UZBeA9n/j6EKSJM3D3cBrqmof4InAK5Lsg/dqSBIw9+T3LOCINnwEcOZowpEkzUdV3VBVX2zDdwBXArvivRqSBAz3qLNTgc8Cj0qyPsmRwLHALyb5BvDMNi5JWkKSrAEeD1zELO7VkKSVbMZOLqrqBdPMOnDEsUiSRiTJA4EPAX9QVd9J8pN5VVVJprxXI8lRwFEAe+yxx2KEKkmLyu6NJWmFSXJfusT3lKr6cJs81L0a3qQsaaUz+ZWkFSTdKd4TgCur6i0Ds7xXQ5IYotmDJGlZeTLwIuDLSS5p0/6U7t6M09t9G9cCh40pPkkaK5NfSVpBquozQKaZ7b0aknrPZg+SJEnqDZNfSZIk9YbJryRJknrD5FeSJEm9YfIrSZKk3jD5lSRJUm+Y/EqSJKk3fM7vJGuOPnvcIUiSJGmBeOZXkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN5bFDW/ehCatbIv5Gb/m2IMXbV+SpKXHM7+SJEnqDZNfSZIk9YbJryRJknrD5FeSJEm9YfIrSZKk3jD5lSRJUm+Y/EqSJKk3TH4lSZLUGya/kiRJ6g2TX0mSJPWGya8kSZJ6w+RXkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN0x+JUmS1Bur5rNykoOAtwNbAcdX1bEjiUqSNHLW2dLiWXP02eMOQdOY85nfJFsB7wSeDewDvCDJPqMKTJI0OtbZktSZT7OH/YF1VXV1Vf0Q+ABwyGjCkiSNmHW2JDG/5HdX4LqB8fVtmiRp6bHOliTm2eZ3GEmOAo5qo99N8rUpFtsJuHmhY5klYxqOMQ3HmIa3oHHlTXNabSdgz9FGsjQNWWfPZFGPrS28p0vpGDeWe1sqccDSiWWpxAFLKJa8aU6xTFtnzyf53QDsPjC+W5u2mao6DjhuSxtKcnFVrZ1HLCNnTMMxpuEY0/CWYlwtpjXjjmOeRlZnz2SpvIdLJQ4wlqUcByydWJZKHLCyY5lPs4fPA3sn2SvJ1sDhwFmjCUuSNGLW2ZLEPM78VtXdSX4P+He6x+acWFVfGVlkkqSRsc6WpM682vxW1ceBj48gjnldYlsgxjQcYxqOMQ1vKca1FGOatRHW2TNZKuW1VOIAY5nKUokDlk4sSyUOWMGxpKpGuT1JkiRpybJ7Y0mSJPXG2JLfJPsmuTDJJUkuTrJ/m54k70iyLsllSfZb5LhemeSrSb6S5P8MTD+mxfS1JL+0mDENxPCaJJVkpzY+trJK8uZWTpcl+UiS7Qfmja2skhzU9rsuydGLue+BGHZPcl6SK9px9Ko2fcck5yT5Rvu/wxhi2yrJl5J8rI3vleSiVl6ntRuhFjOe7ZOc0Y6lK5M8adzllOQP2/t2eZJTk9xv3OW0FCX5n62cfpxk2ruwp/tMjqpMhzlekvxC+66Z+PtBkkPbvPck+ebAvH3nEsewsbTlfjSwv7MGpi9mmeyb5LPtPbwsya8PzJt3mcxUFyfZpr3Gde01rxmYN7LvkCHieHWrqy9Lcm6SPQfmTfk+LWAsL06yaWCfLxuYd0R7P7+R5IhFiOWtA3F8PcltA/NGVi5JTkyyMcnl08xPpslz5lUmVTWWP+ATwLPb8HOA8weG/xUI8ETgokWM6ReATwLbtPGHtv/7AJcC2wB7AVcBWy1yee1Od6PKtcBOS6CsngWsasNvAt407rKiu4nnKuDhwNYtjn0W831qcewC7NeGHwR8vZXL/wGObtOPniizRY7t1cD7gY+18dOBw9vwu4DfWeR4TgJe1oa3BrYfZznRdfrwTeD+A+Xz4nGX01L8A34GeBRwPrB2mmWm/UyOqkxne7wAOwLfBrZt4+8Bnj+iMhkqFuC700xftDIBHgns3YYfBtwAbD+KMhmmLgZ+F3hXGz4cOK0Nj+w7ZMg4fmHgWPidiTi29D4tYCwvBv5hmmP26vZ/hza8w0LGMmn5V9LdILsQ5fI0YD/g8mnmT5nnzLdMxtnsoYAHt+HtgOvb8CHAydW5ENg+yS6LFNPvAMdW1Z0AVbVxIKYPVNWdVfVNYB1dV6GL6a3Aa+nKbcLYyqqqPlFVd7fRC+meGToR07jKakl031pVN1TVF9vwHcCVdEnVIXTJHu3/oYsZV5LdgIOB49t4gAOAM8YRU5Lt6Cq+EwCq6odVdRtjLie6G4Hvn2QVsC1dUjC2clqqqurKqpqpA4wpP5MjPvZme7w8H/jXqvr+HPc3ylh+YrHLpKq+XlXfaMPXAxuB1XPc32TD1MWDMZ4BHNjKYJTfITPGUVXnDRwLg99lozaf76dfAs6pqm9X1a3AOcBBixjLC4BT57G/aVXVBXQ/RqczXZ4zrzIZZ/L7B8Cbk1wH/B1wTJs+zi44Hwk8tV2C+XSSn18CMZHkEGBDVV06adZS6a70pXS/zGC8MS2V8viJdinv8cBFwM5VdUObdSOw8yKH8za6H1A/buMPAW4b+BGz2OW1F7AJeHe6phjHJ3kAYyynqtpAVx99iy7pvR34AuMtp+Vsus/kKI+92R4vh3PvL/I3tkuqb02yzRzjmE0s90vX3O/CtOYXjLFM0jU73JrubOCE+ZTJMHXxT5Zpr/l2ujIYZT0+220dyT3fZTD1+zRXw8bya63cz0gy0SnNqL/bht5eawayF/CpgcmjLJeZTBfrvMpkQbs3TvJJ4KemmPU64EDgD6vqQ0kOozv788yFjGeImFbRnUJ/IvDzwOlJHr7QMQ0R15/SNTNYVFuKqarObMu8DrgbOGUxY1sOkjwQ+BDwB1X1ne6kRqeqKsmiPWolyXOBjVX1hSTPWKz9zmAV3eWuV1bVRUneTneJ9ifGUE470J1p2Au4Dfgg8zvDsqwNUweMO47BkZmOl3bG6GfpmpBNOIYuQdya7nFKfwL85QLHsmdVbWjfL59K8mW65G9oIy6T9wJHVNXED+NZlclKkOQ3gbXA0wcm3+t9qqqrpt7CSHwUOLWq7kzy23Rnxg9YwP0N43DgjKr60cC0xS6XkVvQ5Leqpk1mk5wMvKqNfpB2KZYhu+BcoJh+B/hwdQ1KPpfkx3R9Wy9oTFuKK8nP0n0RX9qSp92AL7Zf6mMrqxbbi4HnAge2MmOhY5rBOPe9mST3pUt8T6mqD7fJNyXZpapuaF84G6ffwsg9GXhekucA96NrcvR2uktIq9qZl8Uur/XA+qq6qI2fQZf8jrOcngl8s6o2AST5MF3ZjbOcxmamOmAI030mb2EWZTpDvT2b4+Uw4CNVddfAtifOkN6Z5N3AH23pBY0ilnaFgaq6Osn5dFeHPsQil0mSBwNn0/2YuXBg27MqkykMUxdPLLO+NTHaju64GGU9PtS2kjyT7kfD0yeaPcK079Nck7wZY6mqWwZGj6druz2x7jMmrXv+HOMYKpYBhwOvmBTnKMtlJtPFOq8yGWezh+u55xfWAcA32vBZwG+1O/yeCNw+8EFcaP9C1/idJI+k+9V7c4vp8HR3p+4F7A18bjECqqovV9VDq2pNVa2hSxj2q6obGWNZJTmI7hL68ya1nRtbWbFEum9t7dZOAK6sqrcMzDoLmLgj9Qhg0c6cVdUxVbVbO4YOBz5VVS8EzqNrAzmOmG4ErkvyqDbpQOAKxlhOdM0dnphk2/Y+TsQ0tnJa5qb8TLYfy6Mq09kcL/dqu9iSw4nP7aHAlHedjyqWJDtMNCNI9+SeJwNXLHaZtPfjI3TtKc+YNG++ZTJMXTwY4/Pp6qRitN8hM8aR5PHAP9F9l20cmD7l+zTHOIaNZfCenefR3S8C3ZWKZ7WYdqC7Ejx49WLksbR4Hk13M9lnB6aNulxmMl2eM78yqRHdsTfbP+ApdG3pLqVrD/lzbXqAd9L9ivgy09xFvEAxbQ28j+6D/kXggIF5r2sxfY32lIoxlds13PO0h3GW1Tq69jaXtL93LYWyorsz9Ott/68b03v0FLobEy8bKJ/n0LVnO5fuh94ngR3HFN8zuOdpDw+n+2JZR3cFZptFjmVf4OJWVv9CV9GOtZyANwBfbfXAe+nuOh9rOS3FP+BX6H6M3wncBPx7m/4w4OMDy035mRxVmU53vNBdwj5+YLk1dGeL7jNp/U+1+vPyVv8/cB5lMmMswP9o+7u0/T9yHGUC/CZw10AddQmw76jKZKr3na7pxPPa8P3aa1zXXvPDB9Yd2XfIEHF8sh2/E2Vw1kzv0wLG8rfAV9o+zwMePbDuS1tZrQNestCxtPHX0z0EYHC9kZYL3Y/RG9qxuJ6u3fXLgZe3+dPmOfMpE3t4kyRJUm/Yw5skSZJ6w+RXkiRJvWHyK0mSpN4w+ZUkSVJvmPxKkiSpN0x+JUmS1Bsmv5IkSeoNk19JkiT1xv8Pw/sd4Osppm4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1, 2, figsize=(12, 5))\n",
    "ax[0].hist(out)\n",
    "ax[0].set_title(\"Distribution of inputs to activation function\\nfor layer with 784 inputs\")\n",
    "ax[1].hist(np.tanh(out))\n",
    "ax[1].set_title(\"Distribution of outputs of tanh function\\nfor layer with 784 inputs\");\n",
    "# fig.savefig(IMG_FILEPATH + \"01_weight_init_activation_illustration.png\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
